sql/flowinfra: TestEvalCtxTxnOnRemoteNodes failed

[cockroach-teamcity]

(sql/flowinfra).TestEvalCtxTxnOnRemoteNodes failed on master@a0236ac92f64c5b582fa9b739cac313ab229bf7f:

I210202 17:45:53.432907 6291 sql/sqlliveness/slstorage/slstorage.go:352  [n1] 311  inserted sqlliveness session 54e61a91c96446a88f3c4e9d9c9d9749
I210202 17:45:53.432958 6291 sql/sqlliveness/slinstance/slinstance.go:144  [n1] 312  created new SQL liveness session 54e61a91c96446a88f3c4e9d9c9d9749
I210202 17:45:53.446105 6299 util/log/event_log.go:32  [n1,intExec=optInToDiagnosticsStatReporting] 313 ={"Timestamp":1612287953435128431,"EventType":"set_cluster_setting","Statement":"SET CLUSTER SETTING \"diagnostics.reporting.enabled\" = true","User":"root","ApplicationName":"$ internal-optInToDiagnosticsStatReporting","SettingName":"diagnostics.reporting.enabled","Value":"true"}
I210202 17:45:53.451223 5232 migration/manager.go:77  [n1,intExec=set-setting,migration-mgr] 314  no need to migrate, cluster already at newest version
I210202 17:45:53.458287 5232 util/log/event_log.go:32  [n1,intExec=set-setting] 315 ={"Timestamp":1612287953449241116,"EventType":"set_cluster_setting","Statement":"SET CLUSTER SETTING version = $1","User":"root","ApplicationName":"$ internal-set-setting","PlaceholderValues":["'20.2-16'"],"SettingName":"version","Value":"20.2-16"}
I210202 17:45:53.531508 6669 util/log/event_log.go:32  [n1,intExec=initializeClusterSecret] 316 ={"Timestamp":1612287953501356293,"EventType":"set_cluster_setting","Statement":"SET CLUSTER SETTING \"cluster.secret\" = gen_random_uuid()::STRING","User":"root","ApplicationName":"$ internal-initializeClusterSecret","SettingName":"cluster.secret","Value":"c81cd56c-555f-46c0-aca6-f0359e702c34"}
I210202 17:45:53.597726 5906 server/server_sql.go:802  [n1] 317  done ensuring all necessary startup migrations have run
I210202 17:45:53.606246 6978 jobs/job_scheduler.go:352  [n1] 318  waiting 4m0s before scheduled jobs daemon start
I210202 17:45:53.607197 5906 server/init.go:196  [n?] 319  no stores initialized
I210202 17:45:53.607249 5906 server/init.go:197  [n?] 320  awaiting `cockroach init` or join with an already initialized node
I210202 17:45:53.608879 6979 server/auto_upgrade.go:55  [n1] 321  no need to upgrade, cluster already at the newest version
I210202 17:45:53.620325 7027 kv/kvserver/liveness/liveness.go:563  [n1] 322  created liveness record for n2
I210202 17:45:53.620397 7027 server/node.go:1265  [n1] 323  allocated IDs: n2, s2
I210202 17:45:53.620694 5906 server/init.go:286  [n?] 324  joined cluster d56745a5-3ea9-49eb-8e57-6c6643ae9027 through join rpc
I210202 17:45:53.620989 5906 server/init.go:287  [n?] 325  received node ID: 2
I210202 17:45:53.621082 5906 server/init.go:288  [n?] 326  received cluster version: 20.2-16
I210202 17:45:53.621208 5906 gossip/gossip.go:402  [n2] 327  NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:43507" > attrs:<> locality:<tiers:<key:"region" value:"test" > tiers:<key:"dc" value:"dc2" > > ServerVersion:<major_val:20 minor_val:2 patch:0 internal:16 > build_tag:"v21.1.0-alpha.1-1591-ga0236ac92f" started_at:1612287953621202218 cluster_name:"" sql_address:<network_field:"tcp" address_field:"127.0.0.1:44927" >
I210202 17:45:53.623450 5906 server/node.go:387  [n2] 328  initialized store s2
I210202 17:45:53.623510 5906 kv/kvserver/stores.go:236  [n2] 329  read 0 node addresses from persistent storage
I210202 17:45:53.623582 5906 server/node.go:464  [n2] 330  started with engine type 2
I210202 17:45:53.623609 5906 server/node.go:466  [n2] 331  started with attributes []
I210202 17:45:53.672656 6822 gossip/client.go:124  [n2] 332  started gossip client to 127.0.0.1:39085
I210202 17:45:53.674417 5984 kv/kvserver/stores.go:255  [n2] 333  wrote 1 node addresses to persistent storage
I210202 17:45:53.679752 5927 kv/kvserver/stores.go:255  [n1] 334  wrote 1 node addresses to persistent storage
I210202 17:45:53.703553 5906 sql/sqlliveness/slinstance/slinstance.go:252  [n2] 335  starting SQL liveness instance
I210202 17:45:53.705409 7288 sql/temporary_schema.go:485  [n2] 336  skipping temporary object cleanup run as it is not the leaseholder
I210202 17:45:53.705518 7288 sql/temporary_schema.go:486  [n2] 337  completed temporary object cleanup job
I210202 17:45:53.705544 7288 sql/temporary_schema.go:610  [n2] 338  temporary object cleaner next scheduled to run at 2021-02-02 18:15:53.70539295 +0000 UTC
I210202 17:45:53.711924 5906 server/server_sql.go:802  [n2] 339  done ensuring all necessary startup migrations have run
I210202 17:45:53.714191 7316 jobs/job_scheduler.go:352  [n2] 340  waiting 2m0s before scheduled jobs daemon start
I210202 17:45:53.719704 7298 sql/sqlliveness/slstorage/slstorage.go:352  [n2] 342  inserted sqlliveness session f5e53cacec6444ab98c59888380c9c6b
I210202 17:45:53.719781 7298 sql/sqlliveness/slinstance/slinstance.go:144  [n2] 343  created new SQL liveness session f5e53cacec6444ab98c59888380c9c6b
I210202 17:45:53.719290 7317 server/auto_upgrade.go:55  [n2] 341  no need to upgrade, cluster already at the newest version
I210202 17:45:53.735804 7206 util/log/event_log.go:32  [n1,client=127.0.0.1:35346,hostssl,user=root] 344 ={"Timestamp":1612287953731526731,"EventType":"set_cluster_setting","Statement":"SET CLUSTER SETTING \"kv.range_merge.queue_enabled\" = false","User":"root","SettingName":"kv.range_merge.queue_enabled","Value":"false"}
    cluster_test.go:657: 
        	Error Trace:	cluster_test.go:657
        	Error:      	Received unexpected error:
        	            	pq: cannot up-replicate to s2; missing gossiped StoreDescriptor
        	Test:       	TestEvalCtxTxnOnRemoteNodes
I210202 17:45:53.787553 5906 util/stop/stopper.go:552  [-] 345  quiescing
I210202 17:45:53.787596 5906 testutils/testcluster/testcluster.go:98  [-] 346  TestCluster quiescing nodes
I210202 17:45:53.787643 7613 util/stop/stopper.go:552  [n2] 347  quiescing
I210202 17:45:53.787731 7612 util/stop/stopper.go:552  [n1] 348  quiescing
W210202 17:45:53.787867 7298 sql/sqlliveness/slinstance/slinstance.go:183  [n2] 349  exiting heartbeat loop
I210202 17:45:53.788179 7168 kv/kvserver/liveness/liveness.go:699  [n2,liveness-hb] 350  unable to get liveness record from KV: unable to get liveness: node unavailable; try another peer
W210202 17:45:53.788458 7299 jobs/registry.go:703  [-] 351  canceling all adopted jobs due to stopper quiescing
W210202 17:45:53.790542 6291 sql/sqlliveness/slinstance/slinstance.go:183  [n1] 352  exiting heartbeat loop
W210202 17:45:53.790706 7601 kv/kvserver/intentresolver/intent_resolver.go:752  [-] 353  failed to gc transaction record: could not GC completed transaction anchored at /Table/SystemConfigSpan/Start: node unavailable; try another peer
W210202 17:45:53.790962 6292 jobs/registry.go:703  [-] 354  canceling all adopted jobs due to stopper quiescing
--- FAIL: TestEvalCtxTxnOnRemoteNodes (0.64s)

More

Parameters:

• GOFLAGS=-json

make stressrace TESTS=TestEvalCtxTxnOnRemoteNodes PKG=./pkg/sql/flowinfra TESTTIMEOUT=5m STRESSFLAGS='-timeout 5m' 2>&1

See this test on roachdash
_{powered by pkg/cmd/internal/issues}