diff --git a/nomad/serf.go b/nomad/serf.go index 14fc24f557b2..f2d62b8cb8f9 100644 --- a/nomad/serf.go +++ b/nomad/serf.go @@ -34,8 +34,9 @@ func (s *Server) serfEventHandler() { case serf.EventMemberLeave, serf.EventMemberFailed: s.nodeFailed(e.(serf.MemberEvent)) s.localMemberEvent(e.(serf.MemberEvent)) - case serf.EventMemberUpdate, serf.EventMemberReap, - serf.EventUser, serf.EventQuery: // Ignore + case serf.EventMemberReap: + s.localMemberEvent(e.(serf.MemberEvent)) + case serf.EventMemberUpdate, serf.EventUser, serf.EventQuery: // Ignore default: s.logger.Printf("[WARN] nomad: unhandled serf event: %#v", e) } diff --git a/nomad/serf_test.go b/nomad/serf_test.go index 21ad70a9034a..c1c85797384e 100644 --- a/nomad/serf_test.go +++ b/nomad/serf_test.go @@ -5,9 +5,11 @@ import ( "fmt" "os" "path" + "strings" "testing" "github.com/hashicorp/nomad/testutil" + "github.com/hashicorp/serf/serf" ) func TestNomad_JoinPeer(t *testing.T) { @@ -88,6 +90,94 @@ func TestNomad_RemovePeer(t *testing.T) { }) } +func TestNomad_ReapPeer(t *testing.T) { + dir := tmpDir(t) + defer os.RemoveAll(dir) + s1 := testServer(t, func(c *Config) { + c.BootstrapExpect = 3 + c.DevMode = false + c.DevDisableBootstrap = true + c.DataDir = path.Join(dir, "node1") + }) + defer s1.Shutdown() + s2 := testServer(t, func(c *Config) { + c.BootstrapExpect = 3 + c.DevMode = false + c.DevDisableBootstrap = true + c.DataDir = path.Join(dir, "node2") + }) + defer s2.Shutdown() + s3 := testServer(t, func(c *Config) { + c.BootstrapExpect = 3 + c.DevMode = false + c.DevDisableBootstrap = true + c.DataDir = path.Join(dir, "node3") + }) + defer s3.Shutdown() + testJoin(t, s1, s2, s3) + + testutil.WaitForResult(func() (bool, error) { + if members := s1.Members(); len(members) != 3 { + return false, fmt.Errorf("bad: %#v", members) + } + if members := s2.Members(); len(members) != 3 { + return false, fmt.Errorf("bad: %#v", members) + } + if members := s3.Members(); len(members) != 3 { + return false, fmt.Errorf("bad: %#v", members) + } + return true, nil + }, func(err error) { + t.Fatalf("err: %v", err) + }) + + testutil.WaitForLeader(t, s1.RPC) + + // Simulate a reap + mems := s1.Members() + var s2mem serf.Member + for _, m := range mems { + if strings.Contains(m.Name, s2.config.NodeName) { + s2mem = m + s2mem.Status = StatusReap + break + } + } + + // Shutdown and then send the reap + s2.Shutdown() + s1.reconcileCh <- s2mem + s2.reconcileCh <- s2mem + s3.reconcileCh <- s2mem + + testutil.WaitForResult(func() (bool, error) { + if len(s1.peers["global"]) != 2 { + return false, fmt.Errorf("bad: %#v", s1.peers["global"]) + } + peers, err := s1.numPeers() + if err != nil { + return false, fmt.Errorf("numPeers() failed: %v", err) + } + if peers != 2 { + return false, fmt.Errorf("bad: %#v", peers) + } + + if len(s3.peers["global"]) != 2 { + return false, fmt.Errorf("bad: %#v", s1.peers["global"]) + } + peers, err = s3.numPeers() + if err != nil { + return false, fmt.Errorf("numPeers() failed: %v", err) + } + if peers != 2 { + return false, fmt.Errorf("bad: %#v", peers) + } + return true, nil + }, func(err error) { + t.Fatalf("err: %v", err) + }) +} + func TestNomad_BootstrapExpect(t *testing.T) { dir := tmpDir(t) defer os.RemoveAll(dir)