keyring: filter by region before checking version

.changelog/14901.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+keyring: Fixed a bug where keyring initialization is blocked by un-upgraded federated regions
+```

nomad/encrypter.go

@@ -474,7 +474,7 @@ START:
 					// new leader has not yet replicated the key from
 					// the old leader before the transition. Ask all
 					// the other servers if they have it.
-					krr.logger.Debug("failed to fetch key from current leader",
+					krr.logger.Warn("failed to fetch key from current leader, trying peers",
 						"key", keyID, "error", err)
 					getReq.AllowStale = true
 					for _, peer := range krr.getAllPeers() {
@@ -494,7 +494,7 @@ START:
 					krr.logger.Error("failed to add key", "key", keyID, "error", err)
 					goto ERR_WAIT
 				}
-				krr.logger.Trace("added key", "key", keyID)
+				krr.logger.Info("added key", "key", keyID)
 			}
 		}
 	}

nomad/leader.go

@@ -1990,7 +1990,15 @@ func (s *Server) initializeKeyring(stopCh <-chan struct{}) {
 			return
 		default:
 		}
-		if ServersMeetMinimumVersion(s.serf.Members(), minVersionKeyring, true) {
+
+		members := s.serf.Members()
+		regionMembers := []serf.Member{}
+		for _, member := range members {
+			if member.Tags["region"] == s.Region() {
+				regionMembers = append(regionMembers, member)
+			}
+		}
+		if ServersMeetMinimumVersion(regionMembers, minVersionKeyring, true) {
 			break
 		}
 	}