From 0a4fc199b186d3529d719214041d23b120b117ec Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Tue, 21 May 2019 15:45:00 -0400 Subject: [PATCH] set node.StatusUpdatedAt in raft Fix a case where `node.StatusUpdatedAt` was manipulated directly in memory. This ensures that StatusUpdatedAt is set in raft layer, and ensures that the field is updated when node drain/eligibility is updated too. --- nomad/drainer_shims.go | 7 ++++++- nomad/fsm.go | 8 ++++---- nomad/node_endpoint.go | 8 +++++++- nomad/state/state_store.go | 17 ++++++++++------- nomad/structs/structs.go | 10 ++++++++++ 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/nomad/drainer_shims.go b/nomad/drainer_shims.go index c9795d5ac4ee..1df9b9aa47e6 100644 --- a/nomad/drainer_shims.go +++ b/nomad/drainer_shims.go @@ -1,6 +1,10 @@ package nomad -import "github.com/hashicorp/nomad/nomad/structs" +import ( + "time" + + "github.com/hashicorp/nomad/nomad/structs" +) // drainerShim implements the drainer.RaftApplier interface required by the // NodeDrainer. @@ -13,6 +17,7 @@ func (d drainerShim) NodesDrainComplete(nodes []string, event *structs.NodeEvent Updates: make(map[string]*structs.DrainUpdate, len(nodes)), NodeEvents: make(map[string]*structs.NodeEvent, len(nodes)), WriteRequest: structs.WriteRequest{Region: d.s.config.Region}, + UpdatedAt: time.Now().Unix(), } update := &structs.DrainUpdate{} diff --git a/nomad/fsm.go b/nomad/fsm.go index 3c91b8f5c694..e2f47783bfc0 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -310,7 +310,7 @@ func (n *nomadFSM) applyStatusUpdate(buf []byte, index uint64) interface{} { panic(fmt.Errorf("failed to decode request: %v", err)) } - if err := n.state.UpdateNodeStatus(index, req.NodeID, req.Status, req.NodeEvent); err != nil { + if err := n.state.UpdateNodeStatus(index, req.NodeID, req.Status, req.UpdatedAt, req.NodeEvent); err != nil { n.logger.Error("UpdateNodeStatus failed", "error", err) return err } @@ -352,7 +352,7 @@ func (n *nomadFSM) applyDrainUpdate(buf []byte, index uint64) interface{} { } } - if err := n.state.UpdateNodeDrain(index, req.NodeID, req.DrainStrategy, req.MarkEligible, req.NodeEvent); err != nil { + if err := n.state.UpdateNodeDrain(index, req.NodeID, req.DrainStrategy, req.MarkEligible, req.UpdatedAt, req.NodeEvent); err != nil { n.logger.Error("UpdateNodeDrain failed", "error", err) return err } @@ -366,7 +366,7 @@ func (n *nomadFSM) applyBatchDrainUpdate(buf []byte, index uint64) interface{} { panic(fmt.Errorf("failed to decode request: %v", err)) } - if err := n.state.BatchUpdateNodeDrain(index, req.Updates, req.NodeEvents); err != nil { + if err := n.state.BatchUpdateNodeDrain(index, req.UpdatedAt, req.Updates, req.NodeEvents); err != nil { n.logger.Error("BatchUpdateNodeDrain failed", "error", err) return err } @@ -387,7 +387,7 @@ func (n *nomadFSM) applyNodeEligibilityUpdate(buf []byte, index uint64) interfac return err } - if err := n.state.UpdateNodeEligibility(index, req.NodeID, req.Eligibility, req.NodeEvent); err != nil { + if err := n.state.UpdateNodeEligibility(index, req.NodeID, req.Eligibility, req.UpdatedAt, req.NodeEvent); err != nil { n.logger.Error("UpdateNodeEligibility failed", "error", err) return err } diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index 5e57406f8a0c..1fed74bbc28f 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -369,7 +369,7 @@ func (n *Node) UpdateStatus(args *structs.NodeUpdateStatusRequest, reply *struct // to track SecretIDs. // Update the timestamp of when the node status was updated - node.StatusUpdatedAt = time.Now().Unix() + args.UpdatedAt = time.Now().Unix() // Commit this update via Raft var index uint64 @@ -484,6 +484,9 @@ func (n *Node) UpdateDrain(args *structs.NodeUpdateDrainRequest, return fmt.Errorf("node not found") } + // Update the timestamp of when the node status was updated + args.UpdatedAt = time.Now().Unix() + // COMPAT: Remove in 0.9. Attempt to upgrade the request if it is of the old // format. if args.Drain && args.DrainStrategy == nil { @@ -589,6 +592,9 @@ func (n *Node) UpdateEligibility(args *structs.NodeUpdateEligibilityRequest, return fmt.Errorf("invalid scheduling eligibility %q", args.Eligibility) } + // Update the timestamp of when the node status was updated + args.UpdatedAt = time.Now().Unix() + // Construct the node event args.NodeEvent = structs.NewNodeEvent().SetSubsystem(structs.NodeEventSubsystemCluster) if node.SchedulingEligibility == args.Eligibility { diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 8bb3b3352e50..cff0aa2347b6 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -732,7 +732,7 @@ func (s *StateStore) DeleteNode(index uint64, nodeID string) error { } // UpdateNodeStatus is used to update the status of a node -func (s *StateStore) UpdateNodeStatus(index uint64, nodeID, status string, event *structs.NodeEvent) error { +func (s *StateStore) UpdateNodeStatus(index uint64, nodeID, status string, updatedAt int64, event *structs.NodeEvent) error { txn := s.db.Txn(true) defer txn.Abort() @@ -748,6 +748,7 @@ func (s *StateStore) UpdateNodeStatus(index uint64, nodeID, status string, event // Copy the existing node existingNode := existing.(*structs.Node) copyNode := existingNode.Copy() + copyNode.StatusUpdatedAt = updatedAt // Add the event if given if event != nil { @@ -771,11 +772,11 @@ func (s *StateStore) UpdateNodeStatus(index uint64, nodeID, status string, event } // BatchUpdateNodeDrain is used to update the drain of a node set of nodes -func (s *StateStore) BatchUpdateNodeDrain(index uint64, updates map[string]*structs.DrainUpdate, events map[string]*structs.NodeEvent) error { +func (s *StateStore) BatchUpdateNodeDrain(index uint64, updatedAt int64, updates map[string]*structs.DrainUpdate, events map[string]*structs.NodeEvent) error { txn := s.db.Txn(true) defer txn.Abort() for node, update := range updates { - if err := s.updateNodeDrainImpl(txn, index, node, update.DrainStrategy, update.MarkEligible, events[node]); err != nil { + if err := s.updateNodeDrainImpl(txn, index, node, update.DrainStrategy, update.MarkEligible, updatedAt, events[node]); err != nil { return err } } @@ -785,11 +786,11 @@ func (s *StateStore) BatchUpdateNodeDrain(index uint64, updates map[string]*stru // UpdateNodeDrain is used to update the drain of a node func (s *StateStore) UpdateNodeDrain(index uint64, nodeID string, - drain *structs.DrainStrategy, markEligible bool, event *structs.NodeEvent) error { + drain *structs.DrainStrategy, markEligible bool, updatedAt int64, event *structs.NodeEvent) error { txn := s.db.Txn(true) defer txn.Abort() - if err := s.updateNodeDrainImpl(txn, index, nodeID, drain, markEligible, event); err != nil { + if err := s.updateNodeDrainImpl(txn, index, nodeID, drain, markEligible, updatedAt, event); err != nil { return err } txn.Commit() @@ -797,7 +798,7 @@ func (s *StateStore) UpdateNodeDrain(index uint64, nodeID string, } func (s *StateStore) updateNodeDrainImpl(txn *memdb.Txn, index uint64, nodeID string, - drain *structs.DrainStrategy, markEligible bool, event *structs.NodeEvent) error { + drain *structs.DrainStrategy, markEligible bool, updatedAt int64, event *structs.NodeEvent) error { // Lookup the node existing, err := txn.First("nodes", "id", nodeID) @@ -811,6 +812,7 @@ func (s *StateStore) updateNodeDrainImpl(txn *memdb.Txn, index uint64, nodeID st // Copy the existing node existingNode := existing.(*structs.Node) copyNode := existingNode.Copy() + copyNode.StatusUpdatedAt = updatedAt // Add the event if given if event != nil { @@ -840,7 +842,7 @@ func (s *StateStore) updateNodeDrainImpl(txn *memdb.Txn, index uint64, nodeID st } // UpdateNodeEligibility is used to update the scheduling eligibility of a node -func (s *StateStore) UpdateNodeEligibility(index uint64, nodeID string, eligibility string, event *structs.NodeEvent) error { +func (s *StateStore) UpdateNodeEligibility(index uint64, nodeID string, eligibility string, updatedAt int64, event *structs.NodeEvent) error { txn := s.db.Txn(true) defer txn.Abort() @@ -857,6 +859,7 @@ func (s *StateStore) UpdateNodeEligibility(index uint64, nodeID string, eligibil // Copy the existing node existingNode := existing.(*structs.Node) copyNode := existingNode.Copy() + copyNode.StatusUpdatedAt = updatedAt // Add the event if given if event != nil { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 33c81dd7e24d..b1d259c9ecf9 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -347,6 +347,7 @@ type NodeUpdateStatusRequest struct { NodeID string Status string NodeEvent *NodeEvent + UpdatedAt int64 WriteRequest } @@ -367,6 +368,9 @@ type NodeUpdateDrainRequest struct { // NodeEvent is the event added to the node NodeEvent *NodeEvent + // UpdatedAt represents server time of receiving request + UpdatedAt int64 + WriteRequest } @@ -379,6 +383,9 @@ type BatchNodeUpdateDrainRequest struct { // NodeEvents is a mapping of the node to the event to add to the node NodeEvents map[string]*NodeEvent + // UpdatedAt represents server time of receiving request + UpdatedAt int64 + WriteRequest } @@ -399,6 +406,9 @@ type NodeUpdateEligibilityRequest struct { // NodeEvent is the event added to the node NodeEvent *NodeEvent + // UpdatedAt represents server time of receiving request + UpdatedAt int64 + WriteRequest }