diff --git a/client/client.go b/client/client.go index 8a51fc073366..d235a5ff614c 100644 --- a/client/client.go +++ b/client/client.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/nomad/client/fingerprint" "github.com/hashicorp/nomad/nomad" "github.com/hashicorp/nomad/nomad/structs" + "github.com/mitchellh/hashstructure" ) const ( @@ -53,6 +54,10 @@ const ( // starting and the intial heartbeat. After the intial heartbeat, // we switch to using the TTL specified by the servers. initialHeartbeatStagger = 10 * time.Second + + // nodeUpdateRetryIntv is how often the client checks for updates to the + // node attributes or meta map. + nodeUpdateRetryIntv = 30 * time.Second ) // DefaultConfig returns the default configuration @@ -604,17 +609,10 @@ func (c *Client) retryIntv(base time.Duration) time.Duration { // run is a long lived goroutine used to run the client func (c *Client) run() { - // Register the client - for { - if err := c.registerNode(); err == nil { - break - } - select { - case <-time.After(c.retryIntv(registerRetryIntv)): - case <-c.shutdownCh: - return - } - } + c.retryRegisterNode() + + // Watch for node changes + go c.watchNodeUpdates() // Setup the heartbeat timer, for the initial registration // we want to do this quickly. We want to do it extra quickly @@ -658,6 +656,42 @@ func (c *Client) run() { } } +// hasNodeChanged calculates a hash for the node attributes- and meta map. +// The new hash values are compared against the old (passed-in) hash values to +// determine if the node properties have changed. It returns the new hash values +// in case they are different from the old hash values. +func (c *Client) hasNodeChanged(oldAttrHash uint64, oldMetaHash uint64) (bool, uint64, uint64) { + newAttrHash, err := hashstructure.Hash(c.config.Node.Attributes, nil) + if err != nil { + c.logger.Printf("[DEBUG] client: unable to calculate node attributes hash: %v", err) + } + // Calculate node meta map hash + newMetaHash, err := hashstructure.Hash(c.config.Node.Meta, nil) + if err != nil { + c.logger.Printf("[DEBUG] client: unable to calculate node meta hash: %v", err) + } + if newAttrHash != oldAttrHash || newMetaHash != oldMetaHash { + return true, newAttrHash, newMetaHash + } + return false, oldAttrHash, oldMetaHash +} + +// retryRegisterNode is used to register the node or update the registration and +// retry in case of failure. +func (c *Client) retryRegisterNode() { + // Register the client + for { + if err := c.registerNode(); err == nil { + break + } + select { + case <-time.After(c.retryIntv(registerRetryIntv)): + case <-c.shutdownCh: + return + } + } +} + // registerNode is used to register the node or update the registration func (c *Client) registerNode() error { node := c.Node() @@ -851,6 +885,25 @@ func (c *Client) watchAllocations(updates chan *allocUpdates) { } } +// watchNodeUpdates periodically checks for changes to the node attributes or meta map +func (c *Client) watchNodeUpdates() { + c.logger.Printf("[DEBUG] client: periodically checking for node changes at duration %v", nodeUpdateRetryIntv) + var attrHash, metaHash uint64 + var changed bool + for { + select { + case <-time.After(nodeUpdateRetryIntv): + changed, attrHash, metaHash = c.hasNodeChanged(attrHash, metaHash) + if changed { + c.logger.Printf("[DEBUG] client: state changed, updating node.") + c.retryRegisterNode() + } + case <-c.shutdownCh: + return + } + } +} + // runAllocs is invoked when we get an updated set of allocations func (c *Client) runAllocs(update *allocUpdates) { // Get the existing allocs diff --git a/client/client_test.go b/client/client_test.go index d1b7bc87a025..6b6c2d27702f 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" + "github.com/mitchellh/hashstructure" ctestutil "github.com/hashicorp/nomad/client/testutil" ) @@ -140,6 +141,37 @@ func TestClient_Fingerprint(t *testing.T) { } } +func TestClient_HasNodeChanged(t *testing.T) { + c := testClient(t, nil) + defer c.Shutdown() + + node := c.Node() + attrHash, err := hashstructure.Hash(node.Attributes, nil) + if err != nil { + c.logger.Printf("[DEBUG] client: unable to calculate node attributes hash: %v", err) + } + // Calculate node meta map hash + metaHash, err := hashstructure.Hash(node.Meta, nil) + if err != nil { + c.logger.Printf("[DEBUG] client: unable to calculate node meta hash: %v", err) + } + if changed, _, _ := c.hasNodeChanged(attrHash, metaHash); changed { + t.Fatalf("Unexpected hash change.") + } + + // Change node attribute + node.Attributes["arch"] = "xyz_86" + if changed, newAttrHash, _ := c.hasNodeChanged(attrHash, metaHash); !changed { + t.Fatalf("Expected hash change in attributes: %d vs %d", attrHash, newAttrHash) + } + + // Change node meta map + node.Meta["foo"] = "bar" + if changed, _, newMetaHash := c.hasNodeChanged(attrHash, metaHash); !changed { + t.Fatalf("Expected hash change in meta map: %d vs %d", metaHash, newMetaHash) + } +} + func TestClient_Fingerprint_InWhitelist(t *testing.T) { c := testClient(t, func(c *config.Config) { if c.Options == nil {