From 449debdf3177807f08ea1afbab823e746e1bed81 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 17 Oct 2022 08:41:17 -0400 Subject: [PATCH 1/2] keyring: filter by region before checking version In #14821 we fixed a panic that can happen if a leadership election happens in the middle of an upgrade. That fix checks that all servers are at the minimum version before initializing the keyring (which blocks evaluation processing during trhe upgrade). But the check we implemented is over the serf membership, which includes servers in any federated regions, which don't necessarily have the same upgrade cycle. Filter the version check by the leader's region. --- .changelog/14901.txt | 3 +++ nomad/leader.go | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .changelog/14901.txt diff --git a/.changelog/14901.txt b/.changelog/14901.txt new file mode 100644 index 000000000000..b36a10b94e53 --- /dev/null +++ b/.changelog/14901.txt @@ -0,0 +1,3 @@ +```release-note:bug +keyring: Fixed a bug where keyring initialization is blocked by un-upgraded federated regions +``` diff --git a/nomad/leader.go b/nomad/leader.go index 4d82fd689dc7..7e9bea28a8fb 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -1990,7 +1990,15 @@ func (s *Server) initializeKeyring(stopCh <-chan struct{}) { return default: } - if ServersMeetMinimumVersion(s.serf.Members(), minVersionKeyring, true) { + + members := s.serf.Members() + regionMembers := []serf.Member{} + for _, member := range members { + if member.Tags["region"] == s.Region() { + regionMembers = append(regionMembers, member) + } + } + if ServersMeetMinimumVersion(regionMembers, minVersionKeyring, true) { break } } From f7523642da2e8be63c6739fe4816c2a830cf3ef9 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 17 Oct 2022 11:38:16 -0400 Subject: [PATCH 2/2] bump up log levels of major keyring operations --- nomad/encrypter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nomad/encrypter.go b/nomad/encrypter.go index 8b3851ff9aed..e6e05bcce79e 100644 --- a/nomad/encrypter.go +++ b/nomad/encrypter.go @@ -474,7 +474,7 @@ START: // new leader has not yet replicated the key from // the old leader before the transition. Ask all // the other servers if they have it. - krr.logger.Debug("failed to fetch key from current leader", + krr.logger.Warn("failed to fetch key from current leader, trying peers", "key", keyID, "error", err) getReq.AllowStale = true for _, peer := range krr.getAllPeers() { @@ -494,7 +494,7 @@ START: krr.logger.Error("failed to add key", "key", keyID, "error", err) goto ERR_WAIT } - krr.logger.Trace("added key", "key", keyID) + krr.logger.Info("added key", "key", keyID) } } }