goroutine profile: total 413 66 @ 0x49c176 0x4ca1cc 0x4ca1ac 0x4e5a2c 0x3ba00b8 0x3b9fa65 0x1568f46 0x4ce321 # 0x4ca1ab sync.runtime_notifyListWait+0x12b GOROOT/src/runtime/sema.go:517 # 0x4e5a2b sync.(*Cond).Wait+0x8b GOROOT/src/sync/cond.go:70 # 0x3ba00b7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker+0x277 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:378 # 0x3b9fa64 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2+0x44 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:321 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 32 @ 0x49c176 0x4ac45c 0x36f1298 0x1568f46 0x4ce321 # 0x36f1297 github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1+0xf7 github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 16 @ 0x49c176 0x465cdb 0x465818 0x16c0c5f 0x62fe43 0x16c0b98 0x4ce321 # labels: {"pebble":"table-cache"} # 0x16c0c5e github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1+0x9e github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:348 # 0x62fe42 runtime/pprof.Do+0xa2 GOROOT/src/runtime/pprof/runtime.go:40 # 0x16c0b97 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop+0x57 github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:346 8 @ 0x49c176 0x4ac45c 0x1793606 0x4ce321 # 0x1793605 github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2+0x85 github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:403 8 @ 0x49c176 0x4ac45c 0x1793a96 0x4ce321 # 0x1793a95 github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1+0x1d5 github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:477 5 @ 0x49c176 0x4ac45c 0x1ca6837 0x1ca5c9a 0x1568f46 0x4ce321 # 0x1ca6836 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run+0x9b6 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 # 0x1ca5c99 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1+0x279 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 4 @ 0x49c176 0x4ac45c 0x156c45c 0x4ce321 # 0x156c45b github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1+0xdb github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:172 4 @ 0x49c176 0x4ac45c 0x36f154d 0x1568f46 0x4ce321 # 0x36f154c github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1+0xcc github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:235 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 4 @ 0x49c176 0x4ac45c 0xd42c7f 0x4ce321 # 0xd42c7e github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon+0x7e github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 3 @ 0x49c176 0x4ac45c 0x330e990 0x1568f46 0x4ce321 # 0x330e98f github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1+0x32f github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:329 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 3 @ 0x49c176 0x4ac45c 0x39e4fc5 0x39e5225 0x39e046c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "n":"1", "r":"7", "rangefeed":"lease", "s":"1"} # 0x39e4fc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop+0x264 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 # 0x39e5224 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop+0xc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 # 0x39e046b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1+0x4b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 3 @ 0x49c176 0x4ac45c 0x39fc4c6 0x1568f46 0x4ce321 # 0x39fc4c5 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run+0x345 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:497 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 3 @ 0x49c176 0x4ac45c 0x83e157 0x83dea5 0x890dfd 0x5706d8 0x890fe5 0x88e436 0x8943af 0x8943b0 0x57637b 0x51729a 0xc2994e 0xc29908 0xc2a1b5 0xc7b514 0xcaa189 0xca98e6 0x4ce321 # 0x83e156 net.(*pipe).read+0x156 GOROOT/src/net/pipe.go:159 # 0x83dea4 net.(*pipe).Read+0x24 GOROOT/src/net/pipe.go:142 # 0x890dfc crypto/tls.(*atLeastReader).Read+0x3c GOROOT/src/crypto/tls/conn.go:787 # 0x5706d7 bytes.(*Buffer).ReadFrom+0x97 GOROOT/src/bytes/buffer.go:202 # 0x890fe4 crypto/tls.(*Conn).readFromUntil+0xe4 GOROOT/src/crypto/tls/conn.go:809 # 0x88e435 crypto/tls.(*Conn).readRecordOrCCS+0x115 GOROOT/src/crypto/tls/conn.go:616 # 0x8943ae crypto/tls.(*Conn).readRecord+0x16e GOROOT/src/crypto/tls/conn.go:582 # 0x8943af crypto/tls.(*Conn).Read+0x16f GOROOT/src/crypto/tls/conn.go:1315 # 0x57637a bufio.(*Reader).Read+0x1ba GOROOT/src/bufio/bufio.go:237 # 0x517299 io.ReadAtLeast+0x99 GOROOT/src/io/io.go:332 # 0xc2994d io.ReadFull+0x6d GOROOT/src/io/io.go:351 # 0xc29907 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 # 0xc2a1b4 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 # 0xc7b513 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams+0x173 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:631 # 0xcaa188 google.golang.org/grpc.(*Server).serveStreams+0x188 google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 # 0xca98e5 google.golang.org/grpc.(*Server).handleRawConn.func1+0x45 google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 3 @ 0x49c176 0x4ac45c 0xc5f755 0xc5fe65 0xc7814e 0x4ce321 # 0xc5f754 google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x114 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:416 # 0xc5fe64 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 # 0xc7814d google.golang.org/grpc/internal/transport.NewServerTransport.func2+0xcd google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:336 3 @ 0x49c176 0x4ac45c 0xc7fb73 0x4ce321 # 0xc7fb72 google.golang.org/grpc/internal/transport.(*http2Server).keepalive+0x232 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1144 2 @ 0x49c176 0x465cdb 0x4657d8 0x1bbbc86 0x1568f46 0x4ce321 # 0x1bbbc85 github.com/cockroachdb/cockroach/pkg/util/netutil.MakeHTTPServer.func2+0x65 github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 2 @ 0x49c176 0x465cdb 0x4657d8 0x1bbc1bd 0x1568f46 0x4ce321 # 0x1bbc1bc github.com/cockroachdb/cockroach/pkg/util/netutil.MakeTCPServer.func1+0x5c github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:126 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 2 @ 0x49c176 0x465cdb 0x465818 0x1630b87 0x163065d 0x62fe43 0x163061c 0x4ce321 # labels: {"pebble":"gc"} # 0x1630b86 github.com/cockroachdb/pebble.(*cleanupManager).mainLoop+0xe6 github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/cleaner.go:155 # 0x163065c github.com/cockroachdb/pebble.openCleanupManager.func1.1+0x1c github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/cleaner.go:86 # 0x62fe42 runtime/pprof.Do+0xa2 GOROOT/src/runtime/pprof/runtime.go:40 # 0x163061b github.com/cockroachdb/pebble.openCleanupManager.func1+0x5b github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/cleaner.go:85 2 @ 0x49c176 0x494777 0x4c85a9 0x53f6f2 0x540a5a 0x540a48 0x826a69 0x839d65 0x890dfd 0x5706d8 0x890fe5 0x88e436 0x8943af 0x8943b0 0x41905d6 0x575d9f 0x57656c 0x280cf6c 0x419c425 0x419bfcb 0x419a956 0x45d61d1 0x45d5ca3 0x45e4b4c 0x1bbca6c 0x1568f46 0x4ce321 # 0x4c85a8 internal/poll.runtime_pollWait+0x88 GOROOT/src/runtime/netpoll.go:305 # 0x53f6f1 internal/poll.(*pollDesc).wait+0x31 GOROOT/src/internal/poll/fd_poll_runtime.go:84 # 0x540a59 internal/poll.(*pollDesc).waitRead+0x259 GOROOT/src/internal/poll/fd_poll_runtime.go:89 # 0x540a47 internal/poll.(*FD).Read+0x247 GOROOT/src/internal/poll/fd_unix.go:167 # 0x826a68 net.(*netFD).Read+0x28 GOROOT/src/net/fd_posix.go:55 # 0x839d64 net.(*conn).Read+0x44 GOROOT/src/net/net.go:183 # 0x890dfc crypto/tls.(*atLeastReader).Read+0x3c GOROOT/src/crypto/tls/conn.go:787 # 0x5706d7 bytes.(*Buffer).ReadFrom+0x97 GOROOT/src/bytes/buffer.go:202 # 0x890fe4 crypto/tls.(*Conn).readFromUntil+0xe4 GOROOT/src/crypto/tls/conn.go:809 # 0x88e435 crypto/tls.(*Conn).readRecordOrCCS+0x115 GOROOT/src/crypto/tls/conn.go:616 # 0x8943ae crypto/tls.(*Conn).readRecord+0x16e GOROOT/src/crypto/tls/conn.go:582 # 0x8943af crypto/tls.(*Conn).Read+0x16f GOROOT/src/crypto/tls/conn.go:1315 # 0x41905d5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read+0x155 github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1488 # 0x575d9e bufio.(*Reader).fill+0xfe GOROOT/src/bufio/bufio.go:106 # 0x57656b bufio.(*Reader).ReadByte+0x2b GOROOT/src/bufio/bufio.go:265 # 0x280cf6b github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg+0x2b github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:215 # 0x419c424 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveImpl.func3+0xa4 github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1034 # 0x419bfca github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveImpl+0x86a github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1187 # 0x419a955 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn+0xb95 github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:803 # 0x45d61d0 github.com/cockroachdb/cockroach/pkg/server.(*systemServerWrapper).serveConn+0x130 github.com/cockroachdb/cockroach/pkg/server/server_controller_sql.go:98 # 0x45d5ca2 github.com/cockroachdb/cockroach/pkg/server.(*serverController).sqlMux+0x2c2 github.com/cockroachdb/cockroach/pkg/server/server_controller_sql.go:78 # 0x45e4b4b github.com/cockroachdb/cockroach/pkg/server.startServeSQL.func1.1+0x22b github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1738 # 0x1bbca6b github.com/cockroachdb/cockroach/pkg/util/netutil.(*TCPServer).ServeWith.func1+0x10b github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:191 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 2 @ 0x49c176 0x4ac45c 0x156de6e 0x4ce321 # 0x156de6d github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked.func1+0x10d github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:575 2 @ 0x49c176 0x4ac45c 0x39e4fc5 0x39e5225 0x39e046c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"9", "dest_s":"1", "n":"1", "r":"9", "rangefeed":"system-config-cache", "s":"1"} # 0x39e4fc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop+0x264 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 # 0x39e5224 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop+0xc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 # 0x39e046b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1+0x4b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 2 @ 0x49c176 0x4ac45c 0x3bb8749 0x1568f46 0x4ce321 # 0x3bb8748 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3+0x2c8 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_gossip.go:147 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 2 @ 0x49c176 0x4ac45c 0x4c888bc 0x1b99564 0x4ce321 # labels: {"job":"CHANGEFEED id=885823367427489793", "n":"1"} # 0x4c888bb github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*teeGroup).tee.func1+0xbb github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/testfeed_test.go:1587 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 2 @ 0x49c176 0x4ac45c 0x9bd38d 0x4ce321 # 0x9bd38c database/sql.(*DB).connectionOpener+0x8c GOROOT/src/database/sql/sql.go:1224 2 @ 0x49c176 0x4ca1cc 0x4ca1ac 0x4e5a2c 0x15c7e85 0x15c7e74 0x62fe43 0x15c781c 0x4ce321 # labels: {"pebble":"wal-sync"} # 0x4ca1ab sync.runtime_notifyListWait+0x12b GOROOT/src/runtime/sema.go:517 # 0x4e5a2b sync.(*Cond).Wait+0x8b GOROOT/src/sync/cond.go:70 # 0x15c7e84 github.com/cockroachdb/pebble/record.(*flusherCond).Wait+0x644 github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:204 # 0x15c7e73 github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop+0x633 github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:429 # 0x62fe42 runtime/pprof.Do+0xa2 GOROOT/src/runtime/pprof/runtime.go:40 # 0x15c781b github.com/cockroachdb/pebble/record.NewLogWriter.func2+0x5b github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:352 2 @ 0x49c176 0x4ca1cc 0x4ca1ac 0x4e5a2c 0x36350c6 0x3600f05 0x3600a56 0x35fb2c6 0x4188de6 0x4ce321 # 0x4ca1ab sync.runtime_notifyListWait+0x12b GOROOT/src/runtime/sema.go:517 # 0x4e5a2b sync.(*Cond).Wait+0x8b GOROOT/src/sync/cond.go:70 # 0x36350c5 github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd+0xc5 github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:539 # 0x3600f04 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd+0xe4 github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2167 # 0x3600a55 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run+0x215 github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2141 # 0x35fb2c5 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn+0xe5 github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:953 # 0x4188de5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1+0x445 github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 1 @ 0x46b474 0x4ca62f 0x610479 0x4ce321 # 0x4ca62e os/signal.signal_recv+0x2e GOROOT/src/runtime/sigqueue.go:152 # 0x610478 os/signal.loop+0x18 GOROOT/src/os/signal/signal_unix.go:23 1 @ 0x490e16 0x4c8085 0x626035 0x625e4d 0x622dab 0x4c81be6 0x4c8a99a 0x4c40d15 0x4c2586e 0x660eab 0x4ce321 # 0x4c8084 runtime/pprof.runtime_goroutineProfileWithLabels+0x24 GOROOT/src/runtime/mprof.go:846 # 0x626034 runtime/pprof.writeRuntimeProfile+0xb4 GOROOT/src/runtime/pprof/pprof.go:723 # 0x625e4c runtime/pprof.writeGoroutine+0x4c GOROOT/src/runtime/pprof/pprof.go:683 # 0x622daa runtime/pprof.(*Profile).WriteTo+0x14a GOROOT/src/runtime/pprof/pprof.go:330 # 0x4c81be5 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*jobFeed).Close+0x4a5 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/testfeed_test.go:575 # 0x4c8a999 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*kafkaFeed).Close+0x79 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/testfeed_test.go:1941 # 0x4c40d14 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.closeFeed+0x34 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/helpers_test.go:744 # 0x4c2586d github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.TestChangefeedKafkaMessageTooLarge.func1.1+0x36d github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_test.go:7719 # 0x660eaa testing.tRunner+0x10a GOROOT/src/testing/testing.go:1446 1 @ 0x49c176 0x465cdb 0x4657d8 0x17863d0 0x17863c6 0x4ce321 # 0x17863cf github.com/cockroachdb/cockroach/pkg/util/admission.(*tokenAllocationTicker).tick+0xaf github.com/cockroachdb/cockroach/pkg/util/admission/io_load_listener.go:334 # 0x17863c5 github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider.func1+0xa5 github.com/cockroachdb/cockroach/pkg/util/admission/grant_coordinator.go:110 1 @ 0x49c176 0x465cdb 0x4657d8 0x1c396c9 0x1568f46 0x4ce321 # 0x1c396c8 github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3+0x48 github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/server.go:388 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x465cdb 0x4657d8 0x23a2365 0x4ce321 # 0x23a2364 github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1+0x124 github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:165 1 @ 0x49c176 0x465cdb 0x4657d8 0x3b9fafb 0x1568f46 0x4ce321 # 0x3b9fafa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1+0x3a github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:290 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x465cdb 0x4657d8 0x458e788 0x1568f46 0x4ce321 # 0x458e787 github.com/cockroachdb/cockroach/pkg/server.configureGRPCGateway.func2+0x47 github.com/cockroachdb/cockroach/pkg/server/grpc_gateway.go:108 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x465cdb 0x4657d8 0x45d81e8 0x1568f46 0x4ce321 # 0x45d81e7 github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func1+0x47 github.com/cockroachdb/cockroach/pkg/server/server_http.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x465cdb 0x4657d8 0x45ec128 0x1568f46 0x4ce321 # 0x45ec127 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func4+0x67 github.com/cockroachdb/cockroach/pkg/server/start_listen.go:138 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x465cdb 0x4657d8 0x45ec2a8 0x1568f46 0x4ce321 # 0x45ec2a7 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func1+0x47 github.com/cockroachdb/cockroach/pkg/server/start_listen.go:77 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x465cdb 0x4657d8 0x661c9a 0x4c2549f 0x4c44ab6 0x4c45187 0x660eab 0x4ce321 # 0x661c99 testing.(*T).Run+0x379 GOROOT/src/testing/testing.go:1494 # 0x4c2549e github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.TestChangefeedKafkaMessageTooLarge.func1+0x1fe github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_test.go:7696 # 0x4c44ab5 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.cdcTestNamed.func1+0x75 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/helpers_test.go:1082 # 0x4c45186 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.cdcTestNamedWithSystem.func1+0x2a6 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/helpers_test.go:1117 # 0x660eaa testing.tRunner+0x10a GOROOT/src/testing/testing.go:1446 1 @ 0x49c176 0x465cdb 0x4657d8 0x661c9a 0x4c44e6a 0x4c449d4 0x4c25b57 0x4c25b0c 0x660eab 0x4ce321 # 0x661c99 testing.(*T).Run+0x379 GOROOT/src/testing/testing.go:1494 # 0x4c44e69 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.cdcTestNamedWithSystem+0x349 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/helpers_test.go:1107 # 0x4c449d3 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.cdcTestNamed+0x93 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/helpers_test.go:1084 # 0x4c25b56 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.cdcTest+0x156 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/helpers_test.go:1077 # 0x4c25b0b github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.TestChangefeedKafkaMessageTooLarge+0x10b github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_test.go:7790 # 0x660eaa testing.tRunner+0x10a GOROOT/src/testing/testing.go:1446 1 @ 0x49c176 0x465cdb 0x4657d8 0x661c9a 0x663bee 0x660eab 0x663a96 0x662579 0x517d505 0x517dcd2 0x49bdb2 0x4ce321 # 0x661c99 testing.(*T).Run+0x379 GOROOT/src/testing/testing.go:1494 # 0x663bed testing.runTests.func1+0x6d GOROOT/src/testing/testing.go:1846 # 0x660eaa testing.tRunner+0x10a GOROOT/src/testing/testing.go:1446 # 0x663a95 testing.runTests+0x455 GOROOT/src/testing/testing.go:1844 # 0x662578 testing.(*M).Run+0x5d8 GOROOT/src/testing/testing.go:1726 # 0x517d504 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl_test.TestMain+0x164 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl_test/pkg/ccl/changefeedccl/main_test.go:31 # 0x517dcd1 main.main+0x471 main/bazel-out/aarch64-opt/bin/pkg/ccl/changefeedccl/changefeedccl_test_/testmain.go:594 # 0x49bdb1 runtime.main+0x211 GOROOT/src/runtime/proc.go:250 1 @ 0x49c176 0x465cdb 0x465818 0x1bba625 0x8c618d 0x93d8a5 0x45d7e68 0x1568f46 0x4ce321 # 0x1bba624 github.com/cockroachdb/cmux.muxListener.Accept+0x44 github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 # 0x8c618c crypto/tls.(*listener).Accept+0x2c GOROOT/src/crypto/tls/tls.go:66 # 0x93d8a4 net/http.(*Server).Serve+0x384 GOROOT/src/net/http/server.go:3070 # 0x45d7e67 github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func4+0x27 github.com/cockroachdb/cockroach/pkg/server/server_http.go:295 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x465cdb 0x465818 0x1bba625 0x93d8a5 0x45d7fa5 0x1568f46 0x4ce321 # 0x1bba624 github.com/cockroachdb/cmux.muxListener.Accept+0x44 github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 # 0x93d8a4 net/http.(*Server).Serve+0x384 GOROOT/src/net/http/server.go:3070 # 0x45d7fa4 github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func3+0x124 github.com/cockroachdb/cockroach/pkg/server/server_http.go:278 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x465cdb 0x465818 0x1bba625 0xca8f15 0x45ebfc8 0x1568f46 0x4ce321 # 0x1bba624 github.com/cockroachdb/cmux.muxListener.Accept+0x44 github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 # 0xca8f14 google.golang.org/grpc.(*Server).Serve+0x474 google.golang.org/grpc/external/org_golang_google_grpc/server.go:823 # 0x45ebfc7 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.1+0x27 github.com/cockroachdb/cockroach/pkg/server/start_listen.go:172 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x465cdb 0x465818 0xd5927a 0x4ce321 # 0xd59279 github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon+0x59 github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 1 @ 0x49c176 0x465cdb 0x465818 0xd594bc 0x4ce321 # 0xd594bb github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher+0xdb github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 1 @ 0x49c176 0x494777 0x4c85a9 0x53f6f2 0x544974 0x544961 0x828bd5 0x8443e8 0x8435bd 0x1bb96a9 0x45d8182 0x1568f46 0x4ce321 # 0x4c85a8 internal/poll.runtime_pollWait+0x88 GOROOT/src/runtime/netpoll.go:305 # 0x53f6f1 internal/poll.(*pollDesc).wait+0x31 GOROOT/src/internal/poll/fd_poll_runtime.go:84 # 0x544973 internal/poll.(*pollDesc).waitRead+0x233 GOROOT/src/internal/poll/fd_poll_runtime.go:89 # 0x544960 internal/poll.(*FD).Accept+0x220 GOROOT/src/internal/poll/fd_unix.go:614 # 0x828bd4 net.(*netFD).accept+0x34 GOROOT/src/net/fd_unix.go:172 # 0x8443e7 net.(*TCPListener).accept+0x27 GOROOT/src/net/tcpsock_posix.go:142 # 0x8435bc net.(*TCPListener).Accept+0x3c GOROOT/src/net/tcpsock.go:288 # 0x1bb96a8 github.com/cockroachdb/cmux.(*cMux).Serve+0xa8 github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:124 # 0x45d8181 github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func2+0x21 github.com/cockroachdb/cockroach/pkg/server/server_http.go:257 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x494777 0x4c85a9 0x53f6f2 0x544974 0x544961 0x828bd5 0x8443e8 0x8435bd 0x1bb96a9 0x45ebf22 0x4e7ac2 0x45ebed5 0x45ebea1 0x1568f46 0x4ce321 # 0x4c85a8 internal/poll.runtime_pollWait+0x88 GOROOT/src/runtime/netpoll.go:305 # 0x53f6f1 internal/poll.(*pollDesc).wait+0x31 GOROOT/src/internal/poll/fd_poll_runtime.go:84 # 0x544973 internal/poll.(*pollDesc).waitRead+0x233 GOROOT/src/internal/poll/fd_poll_runtime.go:89 # 0x544960 internal/poll.(*FD).Accept+0x220 GOROOT/src/internal/poll/fd_unix.go:614 # 0x828bd4 net.(*netFD).accept+0x34 GOROOT/src/net/fd_unix.go:172 # 0x8443e7 net.(*TCPListener).accept+0x27 GOROOT/src/net/tcpsock_posix.go:142 # 0x8435bc net.(*TCPListener).Accept+0x3c GOROOT/src/net/tcpsock.go:288 # 0x1bb96a8 github.com/cockroachdb/cmux.(*cMux).Serve+0xa8 github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:124 # 0x45ebf21 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.3.1+0x21 github.com/cockroachdb/cockroach/pkg/server/start_listen.go:180 # 0x4e7ac1 sync.(*Once).doSlow+0xc1 GOROOT/src/sync/once.go:74 # 0x45ebed4 sync.(*Once).Do+0x54 GOROOT/src/sync/once.go:65 # 0x45ebea0 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.3+0x20 github.com/cockroachdb/cockroach/pkg/server/start_listen.go:179 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x494777 0x4c85a9 0x53f6f2 0x544974 0x544961 0x828bd5 0x8443e8 0x8435bd 0x1bbc69b 0x45e48e6 0x1568f46 0x4ce321 # 0x4c85a8 internal/poll.runtime_pollWait+0x88 GOROOT/src/runtime/netpoll.go:305 # 0x53f6f1 internal/poll.(*pollDesc).wait+0x31 GOROOT/src/internal/poll/fd_poll_runtime.go:84 # 0x544973 internal/poll.(*pollDesc).waitRead+0x233 GOROOT/src/internal/poll/fd_poll_runtime.go:89 # 0x544960 internal/poll.(*FD).Accept+0x220 GOROOT/src/internal/poll/fd_unix.go:614 # 0x828bd4 net.(*netFD).accept+0x34 GOROOT/src/net/fd_unix.go:172 # 0x8443e7 net.(*TCPListener).accept+0x27 GOROOT/src/net/tcpsock_posix.go:142 # 0x8435bc net.(*TCPListener).Accept+0x3c GOROOT/src/net/tcpsock.go:288 # 0x1bbc69a github.com/cockroachdb/cockroach/pkg/util/netutil.(*TCPServer).ServeWith+0x7a github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:166 # 0x45e48e5 github.com/cockroachdb/cockroach/pkg/server.startServeSQL.func1+0xe5 github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1728 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x179b9ed 0x4ce321 # 0x179b9ec github.com/cockroachdb/cockroach/pkg/util/admission.makeStoreWorkQueue.func1+0x8c github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:2155 1 @ 0x49c176 0x4ac45c 0x1b9dd25 0x1568f46 0x4ce321 # 0x1b9dd24 github.com/cockroachdb/cockroach/pkg/security.(*ClientCertExpirationCache).startPurgePastExpirations.func1+0x1a4 github.com/cockroachdb/cockroach/pkg/security/cert_expiry_cache.go:193 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1ba0c14 0x1568f46 0x4ce321 # 0x1ba0c13 github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1+0x113 github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:165 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1bbb25c 0x1bbc69b 0x45c8ef6 0x1568f46 0x4ce321 # 0x1bbb25b github.com/cockroachdb/cockroach/pkg/util/netutil.(*LoopbackListener).Accept+0x9b github.com/cockroachdb/cockroach/pkg/util/netutil/loopback.go:51 # 0x1bbc69a github.com/cockroachdb/cockroach/pkg/util/netutil.(*TCPServer).ServeWith+0x7a github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:166 # 0x45c8ef5 github.com/cockroachdb/cockroach/pkg/server.(*Server).AcceptInternalClients.func1+0x95 github.com/cockroachdb/cockroach/pkg/server/server.go:2207 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1bbb25c 0xca8f15 0x45ebf6b 0x1568f46 0x4ce321 # 0x1bbb25b github.com/cockroachdb/cockroach/pkg/util/netutil.(*LoopbackListener).Accept+0x9b github.com/cockroachdb/cockroach/pkg/util/netutil/loopback.go:51 # 0xca8f14 google.golang.org/grpc.(*Server).Serve+0x474 google.golang.org/grpc/external/org_golang_google_grpc/server.go:823 # 0x45ebf6a github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.2+0x2a github.com/cockroachdb/cockroach/pkg/server/start_listen.go:175 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"10", "dest_s":"1", "n":"1", "rangefeed":"settings-watcher"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"12", "dest_s":"1", "n":"1", "rangefeed":"tenant-capability-watcher"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"22", "dest_s":"1", "n":"1", "rangefeed":"table-stats-cache"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"34", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-protected-ts-records-rangefeed"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"48", "dest_s":"1", "n":"1", "rangefeed":"sql_instances"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"49", "dest_s":"1", "n":"1", "rangefeed":"spanconfig-subscriber"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"51", "dest_s":"1", "n":"1", "rangefeed":"tenant-settings-watcher"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"58", "dest_s":"1", "n":"1", "rangefeed":"tenant-boundaries-watcher"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-descriptor-rangefeed"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "n":"1", "rangefeed":"lease"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "n":"1", "rangefeed":"system-config-cache"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"9", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-zones-rangefeed"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1be0f51 0x4ce321 # labels: {"dest_n":"1", "dest_r":"9", "dest_s":"1", "n":"1", "rangefeed":"system-config-cache"} # 0x1be0f50 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2+0x70 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"10", "dest_s":"1", "n":"1", "rangefeed":"settings-watcher"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"12", "dest_s":"1", "n":"1", "rangefeed":"tenant-capability-watcher"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"22", "dest_s":"1", "n":"1", "rangefeed":"table-stats-cache"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"34", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-protected-ts-records-rangefeed"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"48", "dest_s":"1", "n":"1", "rangefeed":"sql_instances"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"49", "dest_s":"1", "n":"1", "rangefeed":"spanconfig-subscriber"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"51", "dest_s":"1", "n":"1", "rangefeed":"tenant-settings-watcher"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"58", "dest_s":"1", "n":"1", "rangefeed":"tenant-boundaries-watcher"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-descriptor-rangefeed"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "n":"1", "rangefeed":"lease"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "n":"1", "rangefeed":"system-config-cache"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"9", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-zones-rangefeed"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1bf5b88 0x1bf5d65 0x1c08577 0x1be12f7 0x1bf4d59 0x1c7688b 0x1c76fa2 0x1c7601a 0x1c7421b 0x1c71cbc 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"dest_n":"1", "dest_r":"9", "dest_s":"1", "n":"1", "rangefeed":"system-config-cache"} # 0x1bf5b87 github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv+0xe7 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1232 # 0x1bf5d64 github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv+0x44 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1268 # 0x1c08576 github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg+0x76 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1316 # 0x1be12f6 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg+0x36 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 # 0x1bf4d58 github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv+0x58 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1013 # 0x1c7688a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4+0x2a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:773 # 0x1c76fa1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do+0x1a1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 # 0x1c76019 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed+0xc59 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:772 # 0x1c7421a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed+0x85a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:489 # 0x1c71cbb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1+0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:220 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c01513 0x1c007aa 0x1bffeb0 0x1bffbf8 0x1568f46 0x4ce321 # labels: {"tags":"n1,rnode=1,raddr=127.0.0.1:44879,class=default,rpc"} # 0x1c01512 github.com/cockroachdb/cockroach/pkg/rpc.(*peer).runHeartbeatUntilFailure+0x292 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/peer.go:469 # 0x1c007a9 github.com/cockroachdb/cockroach/pkg/rpc.(*peer).runOnce+0x349 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/peer.go:344 # 0x1bffeaf github.com/cockroachdb/cockroach/pkg/rpc.(*peer).run+0x28f github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/peer.go:281 # 0x1bffbf7 github.com/cockroachdb/cockroach/pkg/rpc.(*peer).launch.func1+0x37 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/peer.go:241 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1c01513 0x1c007aa 0x1bffeb0 0x1bffbf8 0x1568f46 0x4ce321 # labels: {"tags":"n1,rnode=1,raddr=127.0.0.1:44879,class=system,rpc"} # 0x1c01512 github.com/cockroachdb/cockroach/pkg/rpc.(*peer).runHeartbeatUntilFailure+0x292 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/peer.go:469 # 0x1c007a9 github.com/cockroachdb/cockroach/pkg/rpc.(*peer).runOnce+0x349 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/peer.go:344 # 0x1bffeaf github.com/cockroachdb/cockroach/pkg/rpc.(*peer).run+0x28f github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/peer.go:281 # 0x1bffbf7 github.com/cockroachdb/cockroach/pkg/rpc.(*peer).launch.func1+0x37 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/peer.go:241 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1c30670 0x1568f46 0x4ce321 # 0x1c3066f github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1+0x26f github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/gossip.go:1252 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1c3104c 0x1568f46 0x4ce321 # 0x1c3104b github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1+0x28b github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/gossip.go:1284 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1c33ac5 0x1568f46 0x4ce321 # 0x1c33ac4 github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1+0x84 github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/infostore.go:192 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-descriptor-rangefeed"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-protected-ts-records-rangefeed"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-zones-rangefeed"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"lease"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"settings-watcher"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"spanconfig-subscriber"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"sql_instances"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"system-config-cache"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"table-stats-cache"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"tenant-boundaries-watcher"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"tenant-capability-watcher"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1c719cb 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"tenant-settings-watcher"} # 0x1c719ca github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1+0x12a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:216 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-descriptor-rangefeed"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-protected-ts-records-rangefeed"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-zones-rangefeed"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"lease"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"settings-watcher"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"spanconfig-subscriber"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"sql_instances"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"system-config-cache"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"table-stats-cache"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"tenant-boundaries-watcher"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"tenant-capability-watcher"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1ca32ff 0x1ca30b8 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"tenant-settings-watcher"} # 0x1ca32fe github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents+0xde github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 # 0x1ca30b7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2+0x37 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x1da7305 0x1568f46 0x4ce321 # 0x1da7304 github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).RefreshLeases.func1+0x104 github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1123 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1da8713 0x1568f46 0x4ce321 # 0x1da8712 github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).PeriodicallyRefreshSomeLeases.func1+0x2b2 github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1250 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1eeb2b6 0x1568f46 0x4ce321 # 0x1eeb2b5 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights.(*concurrentBufferIngester).Start.func2+0x95 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights/pkg/sql/sqlstats/insights/ingester.go:87 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x1eeb3ab 0x1568f46 0x4ce321 # 0x1eeb3aa github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights.(*concurrentBufferIngester).Start.func1+0xca github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights/pkg/sql/sqlstats/insights/ingester.go:70 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x22ee225 0x1568f46 0x4ce321 # 0x22ee224 github.com/cockroachdb/cockroach/pkg/jobs.(*jobScheduler).runDaemon.func1+0x284 github.com/cockroachdb/cockroach/pkg/jobs/job_scheduler.go:417 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x2301e31 0x1568f46 0x4ce321 # 0x2301e30 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10+0x230 github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1202 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x230225a 0x1568f46 0x4ce321 # 0x2302259 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func9+0x1f9 github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1172 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x23026f5 0x1568f46 0x4ce321 # 0x23026f4 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func8+0x1f4 github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1131 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x23289ee 0x1568f46 0x4ce321 # 0x23289ed github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func2+0x28d github.com/cockroachdb/cockroach/pkg/sql/stats/pkg/sql/stats/automatic_stats.go:616 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x2328fa5 0x1568f46 0x4ce321 # 0x2328fa4 github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1+0x224 github.com/cockroachdb/cockroach/pkg/sql/stats/pkg/sql/stats/automatic_stats.go:445 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x23298c9 0x1568f46 0x4ce321 # 0x23298c8 github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1.1+0x1e8 github.com/cockroachdb/cockroach/pkg/sql/stats/pkg/sql/stats/automatic_stats.go:484 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x23749f0 0x1568f46 0x4ce321 # 0x23749ef github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1+0x38f github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:706 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x26b4ddb 0x26b4b14 0x1568f46 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1"} # 0x26b4dda github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).run+0x27a github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:104 # 0x26b4b13 github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).StartReconciler.func1+0x33 github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:77 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x27a6065 0x1568f46 0x4ce321 # 0x27a6064 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).Start.func1+0x284 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:90 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x27b3b6c 0x1568f46 0x4ce321 # 0x27b3b6b github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).startSQLStatsFlushLoop.func1+0x2cb github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/provider.go:172 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x27b4834 0x1568f46 0x4ce321 # 0x27b4833 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*jobMonitor).start.func1+0x313 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/scheduled_job_monitor.go:99 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x27bff8f 0x1568f46 0x4ce321 # 0x27bff8e github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller.(*Controller).Start.func2+0x12e github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller/pkg/sql/catalog/schematelemetry/schematelemetrycontroller/controller.go:117 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x27c29e5 0x1568f46 0x4ce321 # 0x27c29e4 github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache.(*Cache).Start.func1+0xa4 github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache/txn_id_cache.go:145 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x27d6999 0x1568f46 0x4ce321 # 0x27d6998 github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll+0x258 github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics/statement_diagnostics.go:205 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x32fd26b 0x1568f46 0x4ce321 # 0x32fd26a github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startEventIntake.func3+0xaa github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:188 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x32fd7b0 0x1568f46 0x4ce321 # 0x32fd7af github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startResolver.func1+0x20f github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:219 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3306307 0x1568f46 0x4ce321 # 0x3306306 github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run+0x2a6 github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier/notifier.go:166 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3307598 0x1568f46 0x4ce321 # 0x3307597 github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging.(*CaptureIndexUsageStatsLoggingScheduler).start.func1+0x137 github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging/captured_index_usage_stats.go:141 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x36f0e45 0x1568f46 0x4ce321 # 0x36f0e44 github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func4+0x124 github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:192 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3861229 0x2307dac 0x23062ff 0x22e44f2 0x22e3806 0x1568f46 0x4ce321 # labels: {"job":"AUTO UPDATE SQL ACTIVITY id=103", "n":"1"} # 0x3861228 github.com/cockroachdb/cockroach/pkg/sql.(*sqlActivityUpdateJob).Resume+0x268 github.com/cockroachdb/cockroach/pkg/sql/sql_activity_update_job.go:101 # 0x2307dab github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2+0xcb github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1646 # 0x23062fe github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine+0xa3e github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1647 # 0x22e44f1 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob+0x651 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:490 # 0x22e3805 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func2+0x145 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:404 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x387071c 0x1568f46 0x4ce321 # 0x387071b github.com/cockroachdb/cockroach/pkg/sql.(*TemporaryObjectCleaner).Start.func1+0x2bb github.com/cockroachdb/cockroach/pkg/sql/temporary_schema.go:626 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x396529c 0x1568f46 0x4ce321 # 0x396529b github.com/cockroachdb/cockroach/pkg/ccl/utilccl.UpdateMetricOnLicenseChange.func2+0x13b github.com/cockroachdb/cockroach/pkg/ccl/utilccl/license_check.go:169 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x399550a 0x3995394 0x1568f46 0x4ce321 # 0x3995509 github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore.(*SyncWaiterLoop).waitLoop+0x149 github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore/sync_waiter.go:93 # 0x3995393 github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore.(*SyncWaiterLoop).Start.func1+0x33 github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore/sync_waiter.go:84 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39df4df 0x39def7c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"10", "dest_s":"1", "n":"1", "r":"10", "rangefeed":"settings-watcher", "s":"1"} # 0x39df4de github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run+0x51e github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 # 0x39def7b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1+0x3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39df4df 0x39def7c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"12", "dest_s":"1", "n":"1", "r":"12", "rangefeed":"tenant-capability-watcher", "s":"1"} # 0x39df4de github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run+0x51e github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 # 0x39def7b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1+0x3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39df4df 0x39def7c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"22", "dest_s":"1", "n":"1", "r":"22", "rangefeed":"table-stats-cache", "s":"1"} # 0x39df4de github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run+0x51e github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 # 0x39def7b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1+0x3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39df4df 0x39def7c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"34", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "r":"34", "rangefeed":"sql-watcher-protected-ts-records-rangefeed", "s":"1"} # 0x39df4de github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run+0x51e github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 # 0x39def7b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1+0x3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39df4df 0x39def7c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"48", "dest_s":"1", "n":"1", "r":"48", "rangefeed":"sql_instances", "s":"1"} # 0x39df4de github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run+0x51e github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 # 0x39def7b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1+0x3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39df4df 0x39def7c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"49", "dest_s":"1", "n":"1", "r":"49", "rangefeed":"spanconfig-subscriber", "s":"1"} # 0x39df4de github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run+0x51e github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 # 0x39def7b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1+0x3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39df4df 0x39def7c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"51", "dest_s":"1", "n":"1", "r":"51", "rangefeed":"tenant-settings-watcher", "s":"1"} # 0x39df4de github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run+0x51e github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 # 0x39def7b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1+0x3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39df4df 0x39def7c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"58", "dest_s":"1", "n":"1", "r":"58", "rangefeed":"tenant-boundaries-watcher", "s":"1"} # 0x39df4de github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run+0x51e github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 # 0x39def7b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1+0x3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39df4df 0x39def7c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "n":"1", "r":"7", "rangefeed":"lease", "s":"1"} # 0x39df4de github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run+0x51e github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 # 0x39def7b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1+0x3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39df4df 0x39def7c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"9", "dest_s":"1", "n":"1", "r":"9", "rangefeed":"system-config-cache", "s":"1"} # 0x39df4de github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run+0x51e github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 # 0x39def7b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1+0x3b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39e4fc5 0x39e5225 0x39e046c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"10", "dest_s":"1", "n":"1", "r":"10", "rangefeed":"settings-watcher", "s":"1"} # 0x39e4fc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop+0x264 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 # 0x39e5224 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop+0xc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 # 0x39e046b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1+0x4b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39e4fc5 0x39e5225 0x39e046c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"12", "dest_s":"1", "n":"1", "r":"12", "rangefeed":"tenant-capability-watcher", "s":"1"} # 0x39e4fc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop+0x264 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 # 0x39e5224 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop+0xc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 # 0x39e046b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1+0x4b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39e4fc5 0x39e5225 0x39e046c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"22", "dest_s":"1", "n":"1", "r":"22", "rangefeed":"table-stats-cache", "s":"1"} # 0x39e4fc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop+0x264 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 # 0x39e5224 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop+0xc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 # 0x39e046b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1+0x4b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39e4fc5 0x39e5225 0x39e046c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"34", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "r":"34", "rangefeed":"sql-watcher-protected-ts-records-rangefeed", "s":"1"} # 0x39e4fc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop+0x264 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 # 0x39e5224 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop+0xc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 # 0x39e046b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1+0x4b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39e4fc5 0x39e5225 0x39e046c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"48", "dest_s":"1", "n":"1", "r":"48", "rangefeed":"sql_instances", "s":"1"} # 0x39e4fc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop+0x264 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 # 0x39e5224 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop+0xc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 # 0x39e046b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1+0x4b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39e4fc5 0x39e5225 0x39e046c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"49", "dest_s":"1", "n":"1", "r":"49", "rangefeed":"spanconfig-subscriber", "s":"1"} # 0x39e4fc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop+0x264 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 # 0x39e5224 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop+0xc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 # 0x39e046b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1+0x4b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39e4fc5 0x39e5225 0x39e046c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"51", "dest_s":"1", "n":"1", "r":"51", "rangefeed":"tenant-settings-watcher", "s":"1"} # 0x39e4fc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop+0x264 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 # 0x39e5224 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop+0xc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 # 0x39e046b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1+0x4b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x39e4fc5 0x39e5225 0x39e046c 0x1568f46 0x4ce321 # labels: {"dest_n":"1", "dest_r":"58", "dest_s":"1", "n":"1", "r":"58", "rangefeed":"tenant-boundaries-watcher", "s":"1"} # 0x39e4fc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop+0x264 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 # 0x39e5224 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop+0xc4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 # 0x39e046b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1+0x4b github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3a08da9 0x4ce321 # 0x3a08da8 github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission.(*controllerImpl).SetTenantWeightProvider.func1+0xc8 github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission/kvadmission.go:436 1 @ 0x49c176 0x4ac45c 0x3a916e5 0x1568f46 0x4ce321 # 0x3a916e4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).Run.func2+0x1a4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:256 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3aab376 0x1568f46 0x4ce321 # labels: {"n":"1", "split":""} # 0x3aab375 github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc.(*Allocator).start.func1+0x715 github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc/id_alloc.go:151 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3aeee78 0x1568f46 0x4ce321 # labels: {"consistencyChecker":"", "n":"1"} # 0x3aeee77 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2+0x117 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:856 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3aeee78 0x1568f46 0x4ce321 # labels: {"merge":"", "n":"1"} # 0x3aeee77 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2+0x117 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:856 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3aeee78 0x1568f46 0x4ce321 # labels: {"mvccGC":"", "n":"1"} # 0x3aeee77 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2+0x117 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:856 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3aeee78 0x1568f46 0x4ce321 # labels: {"n":"1", "raftlog":""} # 0x3aeee77 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2+0x117 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:856 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3aeee78 0x1568f46 0x4ce321 # labels: {"n":"1", "raftsnapshot":""} # 0x3aeee77 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2+0x117 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:856 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3aeee78 0x1568f46 0x4ce321 # labels: {"n":"1", "replicaGC":""} # 0x3aeee77 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2+0x117 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:856 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3aeee78 0x1568f46 0x4ce321 # labels: {"n":"1", "replicate":""} # 0x3aeee77 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2+0x117 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:856 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3aeee78 0x1568f46 0x4ce321 # labels: {"n":"1", "split":""} # 0x3aeee77 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2+0x117 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:856 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3aeee78 0x1568f46 0x4ce321 # labels: {"n":"1", "timeSeriesMaintenance":""} # 0x3aeee77 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2+0x117 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:856 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3af0eca 0x1568f46 0x4ce321 # labels: {"n":"1", "replicate":""} # 0x3af0ec9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).addToPurgatoryLocked.func2+0x169 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:1197 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3b00a33 0x1568f46 0x4ce321 # 0x3b00a32 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startDroppingFlowTokensForDisconnectedNodes.func1+0x232 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:1046 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3b9dafc 0x3b9e60a 0x3ba4789 0x3b9e25c 0x1568f46 0x4ce321 # 0x3b9dafb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess+0x23b github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:224 # 0x3b9e609 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1.1+0x49 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:286 # 0x3ba4788 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit+0x1e8 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:468 # 0x3b9e25b github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1+0x23b github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:284 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3babcd9 0x1568f46 0x4ce321 # 0x3babcd8 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2+0xb8 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2078 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3bac7bf 0x3bac315 0x1568f46 0x4ce321 # 0x3bac7be github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1.2+0x19e github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2243 # 0x3bac314 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1+0x2f4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2304 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3bc279f 0x1568f46 0x4ce321 # 0x3bc279e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop+0x13e github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:777 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3bc3005 0x1568f46 0x4ce321 # 0x3bc3004 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop+0x104 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:858 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3bc4e09 0x1568f46 0x4ce321 # 0x3bc4e08 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1+0x1a8 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:275 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x3fa43cd 0x4ce321 # 0x3fa43cc go.opencensus.io/stats/view.(*worker).start+0xac go.opencensus.io/stats/view/external/io_opencensus_go/stats/view/worker.go:292 1 @ 0x49c176 0x4ac45c 0x402679b 0x1568f46 0x4ce321 # 0x402679a github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1+0x11a github.com/cockroachdb/cockroach/pkg/ts/db.go:158 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x41cf034 0x1568f46 0x4ce321 # 0x41cf033 github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache.(*Cache).periodicallyRefreshProtectedtsCache+0x233 github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache/cache.go:179 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x41e0196 0x1568f46 0x4ce321 # 0x41e0195 github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports.(*Reporter).Start.func2+0x315 github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports/reporter.go:169 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x41f6705 0x1568f46 0x4ce321 # 0x41f6704 github.com/cockroachdb/cockroach/pkg/server/structlogging.(*hotRangesLoggingScheduler).start.func1+0x204 github.com/cockroachdb/cockroach/pkg/server/structlogging/hot_ranges_log.go:91 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x4201d5e 0x1568f46 0x4ce321 # 0x4201d5d github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber.(*KVSubscriber).Start.func1+0x21d github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go:276 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x44c2185 0x1568f46 0x4ce321 # 0x44c2184 github.com/cockroachdb/cockroach/pkg/util/schedulerlatency.StartSampler.func1+0x424 github.com/cockroachdb/cockroach/pkg/util/schedulerlatency/sampler.go:143 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x44c28ac 0x1568f46 0x4ce321 # 0x44c28ab github.com/cockroachdb/cockroach/pkg/util/schedulerlatency.StartSampler.func1.1+0x2eb github.com/cockroachdb/cockroach/pkg/util/schedulerlatency/sampler.go:108 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x44ef5ee 0x2307dac 0x23062ff 0x22e44f2 0x22e3806 0x1568f46 0x4ce321 # labels: {"job":"AUTO CONFIG RUNNER id=102", "n":"1"} # 0x44ef5ed github.com/cockroachdb/cockroach/pkg/server/autoconfig.(*autoConfigRunner).Resume+0x18d github.com/cockroachdb/cockroach/pkg/server/autoconfig/auto_config.go:75 # 0x2307dab github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2+0xcb github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1646 # 0x23062fe github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine+0xa3e github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1647 # 0x22e44f1 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob+0x651 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:490 # 0x22e3805 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func2+0x145 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:404 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x44f50bb 0x44f4cf0 0x1568f46 0x4ce321 # 0x44f50ba github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).run+0x39a github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:156 # 0x44f4cef github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).Start.func1+0x2f github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:98 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x45041a5 0x4503ae5 0x44fe337 0x44fb6b8 0x41fa1c8 0x2307dac 0x23062ff 0x22e44f2 0x22e3806 0x1568f46 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1"} # 0x45041a4 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher.(*SQLWatcher).watch+0x664 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher/sqlwatcher.go:161 # 0x4503ae4 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher.(*SQLWatcher).WatchForSQLUpdates+0x24 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher/sqlwatcher.go:91 # 0x44fe336 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler.(*incrementalReconciler).reconcile+0xb6 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler/reconciler.go:494 # 0x44fb6b7 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler.(*Reconciler).Reconcile+0x3f7 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler/reconciler.go:188 # 0x41fa1c7 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigjob.(*resumer).Resume+0xb67 github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigjob/job.go:162 # 0x2307dab github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2+0xcb github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1646 # 0x23062fe github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine+0xa3e github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1647 # 0x22e44f1 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob+0x651 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:490 # 0x22e3805 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func2+0x145 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:404 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x450ad68 0x1568f46 0x4ce321 # 0x450ad67 github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop+0x167 github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage/slstorage.go:403 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x4516a07 0x1568f46 0x4ce321 # 0x4516a06 github.com/cockroachdb/cockroach/pkg/sql/sqlinstance/instancestorage.(*Storage).RunInstanceIDReclaimLoop.func2+0x206 github.com/cockroachdb/cockroach/pkg/sql/sqlinstance/instancestorage/instancestorage.go:617 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x451dd4b 0x451d897 0x451e9f4 0x1568f46 0x4ce321 # 0x451dd4a github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoopInner+0x1ca github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:345 # 0x451d896 github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoop+0x56 github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:305 # 0x451e9f3 github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).Start.func1+0x73 github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:442 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x4525fb8 0x2307dac 0x23062ff 0x22e44f2 0x22e3806 0x1568f46 0x4ce321 # labels: {"job":"POLL JOBS STATS id=101", "n":"1"} # 0x4525fb7 github.com/cockroachdb/cockroach/pkg/jobs/metricspoller.(*metricsPoller).Resume+0x217 github.com/cockroachdb/cockroach/pkg/jobs/metricspoller/poller.go:69 # 0x2307dab github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2+0xcb github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1646 # 0x23062fe github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine+0xa3e github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1647 # 0x22e44f1 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob+0x651 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:490 # 0x22e3805 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func2+0x145 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:404 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x4526a49 0x2307dac 0x23062ff 0x22e44f2 0x22e3806 0x1568f46 0x4ce321 # labels: {"job":"KEY VISUALIZER id=100", "n":"1"} # 0x4526a48 github.com/cockroachdb/cockroach/pkg/keyvisualizer/keyvisjob.(*resumer).Resume+0x1c8 github.com/cockroachdb/cockroach/pkg/keyvisualizer/keyvisjob/job.go:64 # 0x2307dab github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2+0xcb github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1646 # 0x23062fe github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine+0xa3e github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1647 # 0x22e44f1 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob+0x651 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:490 # 0x22e3805 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func2+0x145 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:404 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x4571472 0x4571ac9 0x1568f46 0x4ce321 # 0x4571471 github.com/cockroachdb/cockroach/pkg/server.periodicallyPersistHLCUpperBound+0x151 github.com/cockroachdb/cockroach/pkg/server/clock_monotonicity.go:193 # 0x4571ac8 github.com/cockroachdb/cockroach/pkg/server.(*Server).startPersistingHLCUpperBound.func3+0x48 github.com/cockroachdb/cockroach/pkg/server/clock_monotonicity.go:268 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x458b545 0x1568f46 0x4ce321 # 0x458b544 github.com/cockroachdb/cockroach/pkg/server.startSampleEnvironment.func1+0x1c4 github.com/cockroachdb/cockroach/pkg/server/env_sampler.go:146 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x45aa05b 0x1568f46 0x4ce321 # 0x45aa05a github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossiping.func1+0x1da github.com/cockroachdb/cockroach/pkg/server/node.go:887 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x45aa6d7 0x1568f46 0x4ce321 # 0x45aa6d6 github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1+0x176 github.com/cockroachdb/cockroach/pkg/server/node.go:923 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x45ac1a5 0x1568f46 0x4ce321 # 0x45ac1a4 github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func2+0x104 github.com/cockroachdb/cockroach/pkg/server/node.go:1137 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x45d3196 0x1568f46 0x4ce321 # 0x45d3195 github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1+0xf5 github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:49 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x45e67e5 0x1568f46 0x4ce321 # 0x45e67e4 github.com/cockroachdb/cockroach/pkg/server.startSystemLogsGC.func1+0x244 github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:263 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0x4618e65 0x4ce321 # 0x4618e64 github.com/cockroachdb/cockroach/pkg/server.(*TestServer).Start.func1+0xc4 github.com/cockroachdb/cockroach/pkg/server/testserver.go:657 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"10", "dest_s":"1", "n":"1", "r":"10", "rangefeed":"settings-watcher", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"12", "dest_s":"1", "n":"1", "r":"12", "rangefeed":"tenant-capability-watcher", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"22", "dest_s":"1", "n":"1", "r":"22", "rangefeed":"table-stats-cache", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"34", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "r":"34", "rangefeed":"sql-watcher-protected-ts-records-rangefeed", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"48", "dest_s":"1", "n":"1", "r":"48", "rangefeed":"sql_instances", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"49", "dest_s":"1", "n":"1", "r":"49", "rangefeed":"spanconfig-subscriber", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"51", "dest_s":"1", "n":"1", "r":"51", "rangefeed":"tenant-settings-watcher", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"58", "dest_s":"1", "n":"1", "r":"58", "rangefeed":"tenant-boundaries-watcher", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "r":"7", "rangefeed":"sql-watcher-descriptor-rangefeed", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "n":"1", "r":"7", "rangefeed":"lease", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"7", "dest_s":"1", "n":"1", "r":"7", "rangefeed":"system-config-cache", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"9", "dest_s":"1", "job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "r":"9", "rangefeed":"sql-watcher-zones-rangefeed", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4623692 0x45af36b 0x1bf4cb2 0x1bdf9a4 0x1bf3d9b 0x1bf16b4 0x1bf3dc3 0x1be8717 0x1bf3dc3 0x1bf196d 0x15685d1 0x1bf18ef 0x1bf3dc3 0x1bf3ca2 0x1bf4bbb 0x4ce321 # labels: {"dest_n":"1", "dest_r":"9", "dest_s":"1", "n":"1", "r":"9", "rangefeed":"system-config-cache", "s":"1"} # 0x4623691 github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]+0x91 github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 # 0x45af36a github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed+0x1aa github.com/cockroachdb/cockroach/pkg/server/node.go:1628 # 0x1bf4cb1 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1+0x71 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:965 # 0x1bdf9a3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1+0x6c3 github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 # 0x1bf3d9a github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x5a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:780 # 0x1bf16b3 github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4+0x73 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:176 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1be8716 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor+0x3f6 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf196c github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1+0x2c github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:143 # 0x15685d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr+0xd0 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:336 # 0x1bf18ee github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2+0xee github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:142 # 0x1bf3dc2 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1+0x82 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:783 # 0x1bf3ca1 github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run+0x141 github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:785 # 0x1bf4bba github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2+0x11a github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:970 1 @ 0x49c176 0x4ac45c 0x4c888bc 0x1b99564 0x4ce321 # 0x4c888bb github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*teeGroup).tee.func1+0xbb github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/testfeed_test.go:1587 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ac45c 0x4c88f0a 0x4c6b512 0x4c6b635 0x4ba655f 0x4ba5ab4 0x4ce321 # labels: {"job":"CHANGEFEED id=885823367427489793", "n":"1"} # 0x4c88f09 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*fakeKafkaSink).Dial.func3.1+0xe9 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/testfeed_test.go:1712 # 0x4c6b511 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*syncProducerMock).SendMessage+0x31 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/sink_test.go:99 # 0x4c6b634 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*syncProducerMock).SendMessages+0x94 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/sink_test.go:108 # 0x4ba655e github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*kafkaSink).handleBufferedRetries+0x3be github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/sink_kafka.go:653 # 0x4ba5ab3 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*kafkaSink).workerLoop+0x6b3 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/sink_kafka.go:569 1 @ 0x49c176 0x4ac45c 0x83e157 0x83dea5 0x890dfd 0x5706d8 0x890fe5 0x88e436 0x8943af 0x8943b0 0x57637b 0x51729a 0xc2994e 0xc29908 0xc2a1b5 0xc74c05 0x4ce321 # 0x83e156 net.(*pipe).read+0x156 GOROOT/src/net/pipe.go:159 # 0x83dea4 net.(*pipe).Read+0x24 GOROOT/src/net/pipe.go:142 # 0x890dfc crypto/tls.(*atLeastReader).Read+0x3c GOROOT/src/crypto/tls/conn.go:787 # 0x5706d7 bytes.(*Buffer).ReadFrom+0x97 GOROOT/src/bytes/buffer.go:202 # 0x890fe4 crypto/tls.(*Conn).readFromUntil+0xe4 GOROOT/src/crypto/tls/conn.go:809 # 0x88e435 crypto/tls.(*Conn).readRecordOrCCS+0x115 GOROOT/src/crypto/tls/conn.go:616 # 0x8943ae crypto/tls.(*Conn).readRecord+0x16e GOROOT/src/crypto/tls/conn.go:582 # 0x8943af crypto/tls.(*Conn).Read+0x16f GOROOT/src/crypto/tls/conn.go:1315 # 0x57637a bufio.(*Reader).Read+0x1ba GOROOT/src/bufio/bufio.go:237 # 0x517299 io.ReadAtLeast+0x99 GOROOT/src/io/io.go:332 # 0xc2994d io.ReadFull+0x6d GOROOT/src/io/io.go:351 # 0xc29907 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 # 0xc2a1b4 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 # 0xc74c04 google.golang.org/grpc/internal/transport.(*http2Client).reader+0x264 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1597 1 @ 0x49c176 0x4ac45c 0x83e157 0x83dea5 0x890dfd 0x5706d8 0x890fe5 0x88e436 0x8943af 0x8943b0 0x57637b 0x51729a 0xc2994e 0xc29908 0xc2a1b5 0xc74c05 0x4ce321 # labels: {"tags":"n1,rnode=1,raddr=127.0.0.1:44879,class=default,rpc"} # 0x83e156 net.(*pipe).read+0x156 GOROOT/src/net/pipe.go:159 # 0x83dea4 net.(*pipe).Read+0x24 GOROOT/src/net/pipe.go:142 # 0x890dfc crypto/tls.(*atLeastReader).Read+0x3c GOROOT/src/crypto/tls/conn.go:787 # 0x5706d7 bytes.(*Buffer).ReadFrom+0x97 GOROOT/src/bytes/buffer.go:202 # 0x890fe4 crypto/tls.(*Conn).readFromUntil+0xe4 GOROOT/src/crypto/tls/conn.go:809 # 0x88e435 crypto/tls.(*Conn).readRecordOrCCS+0x115 GOROOT/src/crypto/tls/conn.go:616 # 0x8943ae crypto/tls.(*Conn).readRecord+0x16e GOROOT/src/crypto/tls/conn.go:582 # 0x8943af crypto/tls.(*Conn).Read+0x16f GOROOT/src/crypto/tls/conn.go:1315 # 0x57637a bufio.(*Reader).Read+0x1ba GOROOT/src/bufio/bufio.go:237 # 0x517299 io.ReadAtLeast+0x99 GOROOT/src/io/io.go:332 # 0xc2994d io.ReadFull+0x6d GOROOT/src/io/io.go:351 # 0xc29907 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 # 0xc2a1b4 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 # 0xc74c04 google.golang.org/grpc/internal/transport.(*http2Client).reader+0x264 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1597 1 @ 0x49c176 0x4ac45c 0x83e157 0x83dea5 0x890dfd 0x5706d8 0x890fe5 0x88e436 0x8943af 0x8943b0 0x57637b 0x51729a 0xc2994e 0xc29908 0xc2a1b5 0xc74c05 0x4ce321 # labels: {"tags":"n1,rnode=1,raddr=127.0.0.1:44879,class=system,rpc"} # 0x83e156 net.(*pipe).read+0x156 GOROOT/src/net/pipe.go:159 # 0x83dea4 net.(*pipe).Read+0x24 GOROOT/src/net/pipe.go:142 # 0x890dfc crypto/tls.(*atLeastReader).Read+0x3c GOROOT/src/crypto/tls/conn.go:787 # 0x5706d7 bytes.(*Buffer).ReadFrom+0x97 GOROOT/src/bytes/buffer.go:202 # 0x890fe4 crypto/tls.(*Conn).readFromUntil+0xe4 GOROOT/src/crypto/tls/conn.go:809 # 0x88e435 crypto/tls.(*Conn).readRecordOrCCS+0x115 GOROOT/src/crypto/tls/conn.go:616 # 0x8943ae crypto/tls.(*Conn).readRecord+0x16e GOROOT/src/crypto/tls/conn.go:582 # 0x8943af crypto/tls.(*Conn).Read+0x16f GOROOT/src/crypto/tls/conn.go:1315 # 0x57637a bufio.(*Reader).Read+0x1ba GOROOT/src/bufio/bufio.go:237 # 0x517299 io.ReadAtLeast+0x99 GOROOT/src/io/io.go:332 # 0xc2994d io.ReadFull+0x6d GOROOT/src/io/io.go:351 # 0xc29907 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 # 0xc2a1b4 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 # 0xc74c04 google.golang.org/grpc/internal/transport.(*http2Client).reader+0x264 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1597 1 @ 0x49c176 0x4ac45c 0xc5f755 0xc5fe65 0xc69e0d 0x4ce321 # 0xc5f754 google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x114 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:416 # 0xc5fe64 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 # 0xc69e0c google.golang.org/grpc/internal/transport.newHTTP2Client.func6+0x6c google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:448 1 @ 0x49c176 0x4ac45c 0xc5f755 0xc5fe65 0xc69e0d 0x4ce321 # labels: {"tags":"n1,rnode=1,raddr=127.0.0.1:44879,class=default,rpc"} # 0xc5f754 google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x114 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:416 # 0xc5fe64 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 # 0xc69e0c google.golang.org/grpc/internal/transport.newHTTP2Client.func6+0x6c google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:448 1 @ 0x49c176 0x4ac45c 0xc5f755 0xc5fe65 0xc69e0d 0x4ce321 # labels: {"tags":"n1,rnode=1,raddr=127.0.0.1:44879,class=system,rpc"} # 0xc5f754 google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x114 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:416 # 0xc5fe64 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 # 0xc69e0c google.golang.org/grpc/internal/transport.newHTTP2Client.func6+0x6c google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:448 1 @ 0x49c176 0x4ac45c 0xc93c73 0x4ce321 # 0xc93c72 google.golang.org/grpc.(*ccBalancerWrapper).watcher+0x72 google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 1 @ 0x49c176 0x4ac45c 0xc93c73 0x4ce321 # labels: {"tags":"n1,rnode=1,raddr=127.0.0.1:44879,class=default,rpc"} # 0xc93c72 google.golang.org/grpc.(*ccBalancerWrapper).watcher+0x72 google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 1 @ 0x49c176 0x4ac45c 0xc93c73 0x4ce321 # labels: {"tags":"n1,rnode=1,raddr=127.0.0.1:44879,class=system,rpc"} # 0xc93c72 google.golang.org/grpc.(*ccBalancerWrapper).watcher+0x72 google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 1 @ 0x49c176 0x4ac45c 0xc99357 0x1c03dde 0x1568f46 0x4ce321 # labels: {"tags":"n1,rnode=1,raddr=127.0.0.1:44879,class=default,rpc"} # 0xc99356 google.golang.org/grpc.(*ClientConn).WaitForStateChange+0xb6 google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 # 0x1c03ddd github.com/cockroachdb/cockroach/pkg/rpc.launchConnStateWatcher.func1+0x5d github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/peer.go:884 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0xc99357 0x1c03dde 0x1568f46 0x4ce321 # labels: {"tags":"n1,rnode=1,raddr=127.0.0.1:44879,class=system,rpc"} # 0xc99356 google.golang.org/grpc.(*ClientConn).WaitForStateChange+0xb6 google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 # 0x1c03ddd github.com/cockroachdb/cockroach/pkg/rpc.launchConnStateWatcher.func1+0x5d github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/peer.go:884 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0xce6e97 0xce6d35 0x45e1965 0x1568f46 0x4ce321 # 0xce6e96 github.com/cockroachdb/cockroach/pkg/util/tracing.(*Tracer).runPeriodicSnapshotsLoop+0xd6 github.com/cockroachdb/cockroach/pkg/util/tracing/tracer_snapshots.go:284 # 0xce6d34 github.com/cockroachdb/cockroach/pkg/util/tracing.(*Tracer).PeriodicSnapshotsLoop+0xb4 github.com/cockroachdb/cockroach/pkg/util/tracing/tracer_snapshots.go:250 # 0x45e1964 github.com/cockroachdb/cockroach/pkg/server.newSQLServer.func1+0x44 github.com/cockroachdb/cockroach/pkg/server/server_sql.go:547 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ac45c 0xd68fba 0x4ce321 # 0xd68fb9 github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1+0x159 github.com/cockroachdb/cockroach/pkg/util/hlc/pkg/util/hlc/hlc.go:275 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-descriptor-rangefeed"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-protected-ts-records-rangefeed"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-zones-rangefeed"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"rangefeed":"lease"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"rangefeed":"settings-watcher"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"rangefeed":"spanconfig-subscriber"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"rangefeed":"sql_instances"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"rangefeed":"system-config-cache"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"rangefeed":"table-stats-cache"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"rangefeed":"tenant-boundaries-watcher"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"rangefeed":"tenant-capability-watcher"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x1ca2b2d 0x1ca1fb9 0x1568f46 0x4ce321 # labels: {"rangefeed":"tenant-settings-watcher"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x1ca2b2c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run+0x82c github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 # 0x1ca1fb8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2+0x338 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c4ed91 0x4b5cb05 0x4b5b970 0x4b7324b 0x4b71a52 0x4c80235 0x2307dac 0x23062ff 0x22e44f2 0x22f5a98 0x1568f46 0x4ce321 # labels: {"job":"CHANGEFEED id=885823367427489793", "n":"1"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c4ed90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait+0x90 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 # 0x4b5cb04 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.startDistChangefeed+0x544 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_dist.go:308 # 0x4b5b96f github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.distChangefeedFlow+0x2ef github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_dist.go:125 # 0x4b7324a github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changefeedResumer).resumeWithRetries+0x40a github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_stmt.go:1269 # 0x4b71a51 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changefeedResumer).Resume+0x2b1 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_stmt.go:1104 # 0x4c80234 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*reportErrorResumer).Resume+0x34 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/testfeed_test.go:344 # 0x2307dab github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2+0xcb github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1646 # 0x23062fe github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine+0xa3e github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1647 # 0x22e44f1 github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob+0x651 github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:490 # 0x22f5a97 github.com/cockroachdb/cockroach/pkg/jobs.(*StartableJob).Start.func2+0xf7 github.com/cockroachdb/cockroach/pkg/jobs/jobs.go:952 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-descriptor-rangefeed"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-protected-ts-records-rangefeed"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"job":"AUTO SPAN CONFIG RECONCILIATION id=885823351296491521", "n":"1", "rangefeed":"sql-watcher-zones-rangefeed"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"lease"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"settings-watcher"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"spanconfig-subscriber"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"sql_instances"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"system-config-cache"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"table-stats-cache"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"tenant-boundaries-watcher"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"tenant-capability-watcher"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x1b993c7 0x1c4e88a 0x1c71785 0x1c70e55 0x1c9f205 0x1ca3192 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"rangefeed":"tenant-settings-watcher"} # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x1b993c6 golang.org/x/sync/errgroup.(*Group).Wait+0x26 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 # 0x1c4e889 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait+0x49 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 # 0x1c71784 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans+0x5a4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:232 # 0x1c70e54 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed+0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:162 # 0x1c9f204 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed+0x64 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:79 # 0x1ca3191 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1+0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9ec5 0x4e91b2 0x3b9fc05 0x1568f46 0x4ce321 # 0x4c9ec4 sync.runtime_Semacquire+0x24 GOROOT/src/runtime/sema.go:62 # 0x4e91b1 sync.(*WaitGroup).Wait+0x51 GOROOT/src/sync/waitgroup.go:139 # 0x3b9fc04 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait+0x24 github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:332 # 0x1568f45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2+0x145 github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 1 @ 0x49c176 0x4ad51e 0x4ad4f5 0x4c9fe5 0x4e7725 0x4ba4b4a 0x4ba4b36 0x4b9acfd 0x4b9d70a 0x4b63116 0x4b636a5 0x4b6348e 0x4b62fe5 0x4b61ff5 0x4b65d09 0x23ba4d5 0x23bf8f3 0x278d53a 0x36f42d3 0x4b5d015 0x1c4eb65 0x1b99564 0x4ce321 # labels: {"job":"CHANGEFEED id=885823367427489793", "n":"1"} # 0x4c9fe4 sync.runtime_SemacquireMutex+0x24 GOROOT/src/runtime/sema.go:77 # 0x4e7724 sync.(*Mutex).lockSlow+0x164 GOROOT/src/sync/mutex.go:171 # 0x4ba4b49 sync.(*Mutex).Lock+0xc9 GOROOT/src/sync/mutex.go:90 # 0x4ba4b35 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*kafkaSink).Flush+0xb5 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/sink_kafka.go:416 # 0x4b9acfc github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.errorWrapperSink.Flush+0x5c github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/sink.go:432 # 0x4b9d709 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*safeSink).Flush+0xa9 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/sink.go:702 # 0x4b63115 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).flushBufferedEvents+0x95 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:670 # 0x4b636a4 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).flushFrontier+0x24 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:724 # 0x4b6348d github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).noteResolvedSpan+0x34d github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:698 # 0x4b62fe4 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).tick+0x484 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:658 # 0x4b61ff4 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeAggregator).Next+0xd4 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:578 # 0x4b65d08 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.(*changeFrontier).Next+0xe8 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_processors.go:1231 # 0x23ba4d4 github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run+0x54 github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:196 # 0x23bf8f2 github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run+0x72 github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:726 # 0x278d539 github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run+0x219 github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:575 # 0x36f42d2 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run+0xb92 github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:902 # 0x4b5d014 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl.startDistChangefeed.func1+0x414 github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/changefeed_dist.go:304 # 0x1c4eb64 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1+0x24 github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 # 0x1b99563 golang.org/x/sync/errgroup.(*Group).Go.func1+0x63 golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75