-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
core: track node plan rejections and mark clients as ineligible
- Loading branch information
Showing
5 changed files
with
290 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| package nomad | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "github.com/hashicorp/go-hclog" | ||
| lru "github.com/hashicorp/golang-lru" | ||
| ) | ||
|
|
||
| // BadNodeTracker keeps a record of nodes marked as bad by the plan applier. | ||
| // | ||
| // It takes a time window and a threshold score value. Node scores are | ||
| // incremented every time they are added to the tracker. If the score is above | ||
| // the threshold within the time window the node is reported as bad. | ||
| // | ||
| // The tracker uses a fixed size ARC cache that evicts old entries based on | ||
| // access frequency and recency. | ||
| type BadNodeTracker struct { | ||
| logger hclog.Logger | ||
| cache *lru.ARCCache | ||
| window time.Duration | ||
| threshold int | ||
| } | ||
|
|
||
| // NewBadNodeTracker returns a new BadNodeTracker. | ||
| func NewBadNodeTracker(logger hclog.Logger, size int, window time.Duration, threshold int) (*BadNodeTracker, error) { | ||
| cache, err := lru.NewARC(size) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to create new bad node tracker: %v", err) | ||
| } | ||
|
|
||
| return &BadNodeTracker{ | ||
| logger: logger.Named("bad_node_tracker"). | ||
| With("threshold", threshold). | ||
| With("window", window), | ||
| cache: cache, | ||
| window: window, | ||
| threshold: threshold, | ||
| }, nil | ||
| } | ||
|
|
||
| // IsBad returns true if the node has a score above the threshold within the | ||
| // time window. | ||
| func (t *BadNodeTracker) IsBad(nodeID string) bool { | ||
| value, ok := t.cache.Get(nodeID) | ||
| if !ok { | ||
| return false | ||
| } | ||
|
|
||
| stats := value.(*badNodeStats) | ||
| score := stats.score(time.Now()) | ||
|
|
||
| t.logger.Debug("checking if node is bad", "node_id", nodeID, "score", score) | ||
| return score > t.threshold | ||
| } | ||
|
|
||
| // Add adds a node to be tracked. If the node is already being tracked its | ||
| // score is incremented. | ||
| func (t *BadNodeTracker) Add(nodeID string) { | ||
| value, ok := t.cache.Get(nodeID) | ||
| if !ok { | ||
| value = newBadNodeStats(t.window) | ||
| t.cache.Add(nodeID, value) | ||
| } | ||
|
|
||
| stats := value.(*badNodeStats) | ||
| score := stats.incrementScore() | ||
| t.logger.Debug("marking node as bad", "node_id", nodeID, "score", score) | ||
| } | ||
|
|
||
| // badNodeStats represents a node being tracked by BadNodeTracker. | ||
| type badNodeStats struct { | ||
| count int | ||
| ts time.Time | ||
| window time.Duration | ||
| } | ||
|
|
||
| func newBadNodeStats(window time.Duration) *badNodeStats { | ||
| return &badNodeStats{ | ||
| count: 0, | ||
| ts: time.Now(), | ||
| window: window, | ||
| } | ||
| } | ||
|
|
||
| // score returns the current count for the node or 0 if the time window has | ||
| // expired. | ||
| func (s *badNodeStats) score(t time.Time) int { | ||
| expiry := s.ts.Add(s.window) | ||
| if t.After(expiry) { | ||
| s.count = 0 | ||
| } | ||
| return s.count | ||
| } | ||
|
|
||
| // incrementScore adds 1 to the score and updates the timestamp to move the | ||
| // expiration date forward. | ||
| func (s *badNodeStats) incrementScore() int { | ||
| s.count += 1 | ||
| s.ts = time.Now() | ||
| return s.count | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| package nomad | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/hashicorp/go-hclog" | ||
| "github.com/hashicorp/nomad/ci" | ||
| "github.com/hashicorp/nomad/testutil" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestBadNodeTracker(t *testing.T) { | ||
| ci.Parallel(t) | ||
|
|
||
| cacheSize := 3 | ||
| tracker, err := NewBadNodeTracker( | ||
| hclog.NewNullLogger(), cacheSize, time.Second, 10) | ||
| require.NoError(t, err) | ||
|
|
||
| for i := 0; i < 10; i++ { | ||
| tracker.Add(fmt.Sprintf("node-%d", i+1)) | ||
| } | ||
|
|
||
| require.Equal(t, cacheSize, tracker.cache.Len()) | ||
|
|
||
| // Only track the most recent values. | ||
| expected := []string{"node-8", "node-9", "node-10"} | ||
| require.ElementsMatch(t, expected, tracker.cache.Keys()) | ||
| } | ||
|
|
||
| func TestBadNodeTracker_IsBad(t *testing.T) { | ||
| ci.Parallel(t) | ||
|
|
||
| window := time.Duration(testutil.TestMultiplier()) * time.Second | ||
| tracker, err := NewBadNodeTracker(hclog.NewNullLogger(), 3, window, 4) | ||
| require.NoError(t, err) | ||
|
|
||
| // Populate cache. | ||
| tracker.Add("node-1") | ||
|
|
||
| tracker.Add("node-2") | ||
| tracker.Add("node-2") | ||
|
|
||
| tracker.Add("node-3") | ||
| tracker.Add("node-3") | ||
| tracker.Add("node-3") | ||
| tracker.Add("node-3") | ||
| tracker.Add("node-3") | ||
| tracker.Add("node-3") | ||
|
|
||
| testCases := []struct { | ||
| name string | ||
| nodeID string | ||
| bad bool | ||
| }{ | ||
| { | ||
| name: "node-1 is not bad", | ||
| nodeID: "node-1", | ||
| bad: false, | ||
| }, | ||
| { | ||
| name: "node-3 is bad", | ||
| nodeID: "node-3", | ||
| bad: true, | ||
| }, | ||
| { | ||
| name: "node not tracked is not bad", | ||
| nodeID: "node-1000", | ||
| bad: false, | ||
| }, | ||
| } | ||
|
|
||
| for _, tc := range testCases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| got := tracker.IsBad(tc.nodeID) | ||
| require.Equal(t, tc.bad, got) | ||
| }) | ||
| } | ||
|
|
||
| t.Run("cache expires", func(t *testing.T) { | ||
| time.Sleep(window) | ||
| require.False(t, tracker.IsBad("node-1")) | ||
| require.False(t, tracker.IsBad("node-2")) | ||
| require.False(t, tracker.IsBad("node-3")) | ||
| }) | ||
|
|
||
| t.Run("IsBad updates cache", func(t *testing.T) { | ||
| // Don't access node-3 so it should be evicted when a new value is | ||
| // added and the tracker size overflows. | ||
| tracker.IsBad("node-1") | ||
| tracker.IsBad("node-2") | ||
| tracker.Add("node-4") | ||
|
|
||
| expected := []string{"node-1", "node-2", "node-4"} | ||
| require.ElementsMatch(t, expected, tracker.cache.Keys()) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.