From 035cc02c6551a6b9bbbc813bc222585375e44532 Mon Sep 17 00:00:00 2001 From: Ivo Verberk Date: Wed, 3 Feb 2016 21:07:09 +0100 Subject: [PATCH 1/2] Reregister node when periodic fingerprint changes node properties --- client/client.go | 72 ++++++++++++++++++++++++++++++++++++------- client/client_test.go | 32 +++++++++++++++++++ 2 files changed, 93 insertions(+), 11 deletions(-) diff --git a/client/client.go b/client/client.go index 8a51fc073366..10e3eb10e1d6 100644 --- a/client/client.go +++ b/client/client.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/nomad/client/fingerprint" "github.com/hashicorp/nomad/nomad" "github.com/hashicorp/nomad/nomad/structs" + "github.com/mitchellh/hashstructure" ) const ( @@ -53,6 +54,10 @@ const ( // starting and the intial heartbeat. After the intial heartbeat, // we switch to using the TTL specified by the servers. initialHeartbeatStagger = 10 * time.Second + + // nodeUpdateRetryIntv is how often the client checks for updates to the + // node attributes or meta map. + nodeUpdateRetryIntv = 30 * time.Second ) // DefaultConfig returns the default configuration @@ -604,17 +609,10 @@ func (c *Client) retryIntv(base time.Duration) time.Duration { // run is a long lived goroutine used to run the client func (c *Client) run() { - // Register the client - for { - if err := c.registerNode(); err == nil { - break - } - select { - case <-time.After(c.retryIntv(registerRetryIntv)): - case <-c.shutdownCh: - return - } - } + c.retryRegisterNode() + + // Watch for node changes + go c.watchNodeUpdates() // Setup the heartbeat timer, for the initial registration // we want to do this quickly. We want to do it extra quickly @@ -658,6 +656,39 @@ func (c *Client) run() { } } +func (c *Client) hasNodeChanged(oldAttrHash uint64, oldMetaHash uint64) (bool, uint64, uint64) { + newAttrHash, err := hashstructure.Hash(c.config.Node.Attributes, nil) + if err != nil { + c.logger.Printf("[DEBUG] client: unable to calculate node attributes hash: %v", err) + } + // Calculate node meta map hash + newMetaHash, err := hashstructure.Hash(c.config.Node.Meta, nil) + if err != nil { + c.logger.Printf("[DEBUG] client: unable to calculate node meta hash: %v", err) + } + if newAttrHash != oldAttrHash || newMetaHash != oldMetaHash { + return true, newAttrHash, newMetaHash + } else { + return false, oldAttrHash, oldMetaHash + } +} + +// retryRegisterNode is used to register the node or update the registration and +// retry in case of failure. +func (c *Client) retryRegisterNode() { + // Register the client + for { + if err := c.registerNode(); err == nil { + break + } + select { + case <-time.After(c.retryIntv(registerRetryIntv)): + case <-c.shutdownCh: + return + } + } +} + // registerNode is used to register the node or update the registration func (c *Client) registerNode() error { node := c.Node() @@ -851,6 +882,25 @@ func (c *Client) watchAllocations(updates chan *allocUpdates) { } } +// watchNodeUpdates periodically checks for changes to the node attributes or meta map +func (c *Client) watchNodeUpdates() { + c.logger.Printf("[DEBUG] client: periodically checking for node changes at duration %v", nodeUpdateRetryIntv) + var attrHash, metaHash uint64 + var changed bool + for { + select { + case <-time.After(nodeUpdateRetryIntv): + changed, attrHash, metaHash = c.hasNodeChanged(attrHash, metaHash) + if changed { + c.logger.Printf("[DEBUG] client: state changed, updating node.") + c.retryRegisterNode() + } + case <-c.shutdownCh: + return + } + } +} + // runAllocs is invoked when we get an updated set of allocations func (c *Client) runAllocs(update *allocUpdates) { // Get the existing allocs diff --git a/client/client_test.go b/client/client_test.go index d1b7bc87a025..6b6c2d27702f 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" + "github.com/mitchellh/hashstructure" ctestutil "github.com/hashicorp/nomad/client/testutil" ) @@ -140,6 +141,37 @@ func TestClient_Fingerprint(t *testing.T) { } } +func TestClient_HasNodeChanged(t *testing.T) { + c := testClient(t, nil) + defer c.Shutdown() + + node := c.Node() + attrHash, err := hashstructure.Hash(node.Attributes, nil) + if err != nil { + c.logger.Printf("[DEBUG] client: unable to calculate node attributes hash: %v", err) + } + // Calculate node meta map hash + metaHash, err := hashstructure.Hash(node.Meta, nil) + if err != nil { + c.logger.Printf("[DEBUG] client: unable to calculate node meta hash: %v", err) + } + if changed, _, _ := c.hasNodeChanged(attrHash, metaHash); changed { + t.Fatalf("Unexpected hash change.") + } + + // Change node attribute + node.Attributes["arch"] = "xyz_86" + if changed, newAttrHash, _ := c.hasNodeChanged(attrHash, metaHash); !changed { + t.Fatalf("Expected hash change in attributes: %d vs %d", attrHash, newAttrHash) + } + + // Change node meta map + node.Meta["foo"] = "bar" + if changed, _, newMetaHash := c.hasNodeChanged(attrHash, metaHash); !changed { + t.Fatalf("Expected hash change in meta map: %d vs %d", metaHash, newMetaHash) + } +} + func TestClient_Fingerprint_InWhitelist(t *testing.T) { c := testClient(t, func(c *config.Config) { if c.Options == nil { From a78836049f51b4742a05d5984b1b9a131ad00c56 Mon Sep 17 00:00:00 2001 From: Ivo Verberk Date: Thu, 4 Feb 2016 08:17:53 +0100 Subject: [PATCH 2/2] Add comments to hasNodeChanged and remove superfluous else block --- client/client.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/client.go b/client/client.go index 10e3eb10e1d6..d235a5ff614c 100644 --- a/client/client.go +++ b/client/client.go @@ -656,6 +656,10 @@ func (c *Client) run() { } } +// hasNodeChanged calculates a hash for the node attributes- and meta map. +// The new hash values are compared against the old (passed-in) hash values to +// determine if the node properties have changed. It returns the new hash values +// in case they are different from the old hash values. func (c *Client) hasNodeChanged(oldAttrHash uint64, oldMetaHash uint64) (bool, uint64, uint64) { newAttrHash, err := hashstructure.Hash(c.config.Node.Attributes, nil) if err != nil { @@ -668,9 +672,8 @@ func (c *Client) hasNodeChanged(oldAttrHash uint64, oldMetaHash uint64) (bool, u } if newAttrHash != oldAttrHash || newMetaHash != oldMetaHash { return true, newAttrHash, newMetaHash - } else { - return false, oldAttrHash, oldMetaHash } + return false, oldAttrHash, oldMetaHash } // retryRegisterNode is used to register the node or update the registration and