Nodes generate Secret ID and used retrieving allocations/registering

.travis.yml

@@ -16,7 +16,6 @@ matrix:
 branches:
   only:
     - master
-    - /^f-.*$/
 
 before_install:  
     - sudo apt-get update

client/client.go

@@ -486,33 +486,54 @@ func (c *Client) getAllocRunners() map[string]*AllocRunner {
 	return runners
 }
 
-// nodeID restores a persistent unique ID or generates a new one
-func (c *Client) nodeID() (string, error) {
+// nodeIDs restores the nodes persistent unique ID and SecretID or generates new
+// ones
+func (c *Client) nodeID() (id string, secret string, err error) {
 	// Do not persist in dev mode
 	if c.config.DevMode {
-		return structs.GenerateUUID(), nil
+		return structs.GenerateUUID(), structs.GenerateUUID(), nil
 	}
 
 	// Attempt to read existing ID
-	path := filepath.Join(c.config.StateDir, "client-id")
-	buf, err := ioutil.ReadFile(path)
+	idPath := filepath.Join(c.config.StateDir, "client-id")
+	idBuf, err := ioutil.ReadFile(idPath)
 	if err != nil && !os.IsNotExist(err) {
-		return "", err
+		return "", "", err
+	}
+
+	// Attempt to read existing secret ID
+	secretPath := filepath.Join(c.config.StateDir, "secret-id")
+	secretBuf, err := ioutil.ReadFile(secretPath)
+	if err != nil && !os.IsNotExist(err) {
+		return "", "", err
 	}
 
 	// Use existing ID if any
-	if len(buf) != 0 {
-		return string(buf), nil
+	if len(idBuf) != 0 {
+		id = string(idBuf)
+	} else {
+		// Generate new ID
+		id = structs.GenerateUUID()
+
+		// Persist the ID
+		if err := ioutil.WriteFile(idPath, []byte(id), 0700); err != nil {
+			return "", "", err
+		}
 	}
 
-	// Generate new ID
-	id := structs.GenerateUUID()
+	if len(secretBuf) != 0 {
+		secret = string(secretBuf)
+	} else {
+		// Generate new ID
+		secret = structs.GenerateUUID()
 
-	// Persist the ID
-	if err := ioutil.WriteFile(path, []byte(id), 0700); err != nil {
-		return "", err
+		// Persist the ID
+		if err := ioutil.WriteFile(secretPath, []byte(secret), 0700); err != nil {
+			return "", "", err
+		}
 	}
-	return id, nil
+
+	return id, secret, nil
 }
 
 // setupNode is used to setup the initial node
@@ -523,11 +544,13 @@ func (c *Client) setupNode() error {
 		c.config.Node = node
 	}
 	// Generate an iD for the node
-	var err error
-	node.ID, err = c.nodeID()
+	id, secretID, err := c.nodeID()
 	if err != nil {
 		return fmt.Errorf("node ID setup failed: %v", err)
 	}
+
+	node.ID = id
+	node.SecretID = secretID
 	if node.Attributes == nil {
 		node.Attributes = make(map[string]string)
 	}
@@ -827,6 +850,8 @@ func (c *Client) retryRegisterNode() {
 	for {
 		if err := c.registerNode(); err == nil {
 			break
+		} else {
+			c.logger.Printf("[ERR] client: %v", err)
 		}
 		select {
 		case <-time.After(c.retryIntv(registerRetryIntv)):
@@ -992,8 +1017,10 @@ func (c *Client) watchAllocations(updates chan *allocUpdates) {
 	// The request and response for getting the map of allocations that should
 	// be running on the Node to their AllocModifyIndex which is incremented
 	// when the allocation is updated by the servers.
+	n := c.Node()
 	req := structs.NodeSpecificRequest{
-		NodeID: c.Node().ID,
+		NodeID:   n.ID,
+		SecretID: n.SecretID,
 		QueryOptions: structs.QueryOptions{
 			Region:     c.Region(),
 			AllowStale: true,
@@ -1417,10 +1444,7 @@ func (c *Client) collectHostStats() {
 
 // emitStats pushes host resource usage stats to remote metrics collection sinks
 func (c *Client) emitStats(hStats *stats.HostStats) {
-	nodeID, err := c.nodeID()
-	if err != nil {
-		return
-	}
+	nodeID := c.Node().ID
 	metrics.SetGauge([]string{"client", "host", "memory", nodeID, "total"}, float32(hStats.Memory.Total))
 	metrics.SetGauge([]string{"client", "host", "memory", nodeID, "available"}, float32(hStats.Memory.Available))
 	metrics.SetGauge([]string{"client", "host", "memory", nodeID, "used"}, float32(hStats.Memory.Used))

client/client_test.go

@@ -352,16 +352,26 @@ func TestClient_UpdateAllocStatus(t *testing.T) {
 	})
 	defer c1.Shutdown()
 
+	// Wait til the node is ready
+	waitTilNodeReady(c1, t)
+
+	job := mock.Job()
 	alloc := mock.Alloc()
 	alloc.NodeID = c1.Node().ID
+	alloc.Job = job
+	alloc.JobID = job.ID
 	originalStatus := "foo"
 	alloc.ClientStatus = originalStatus
 
+	// Insert at zero so they are pulled
 	state := s1.State()
-	if err := state.UpsertJobSummary(99, mock.JobSummary(alloc.JobID)); err != nil {
+	if err := state.UpsertJob(0, job); err != nil {
+		t.Fatal(err)
+	}
+	if err := state.UpsertJobSummary(100, mock.JobSummary(alloc.JobID)); err != nil {
 		t.Fatal(err)
 	}
-	state.UpsertAllocs(100, []*structs.Allocation{alloc})
+	state.UpsertAllocs(101, []*structs.Allocation{alloc})
 
 	testutil.WaitForResult(func() (bool, error) {
 		out, err := state.AllocByID(alloc.ID)
@@ -391,21 +401,29 @@ func TestClient_WatchAllocs(t *testing.T) {
 	})
 	defer c1.Shutdown()
 
+	// Wait til the node is ready
+	waitTilNodeReady(c1, t)
+
 	// Create mock allocations
+	job := mock.Job()
 	alloc1 := mock.Alloc()
+	alloc1.JobID = job.ID
+	alloc1.Job = job
 	alloc1.NodeID = c1.Node().ID
 	alloc2 := mock.Alloc()
 	alloc2.NodeID = c1.Node().ID
+	alloc2.JobID = job.ID
+	alloc2.Job = job
 
+	// Insert at zero so they are pulled
 	state := s1.State()
-	if err := state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)); err != nil {
+	if err := state.UpsertJob(100, job); err != nil {
 		t.Fatal(err)
 	}
-	if err := state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)); err != nil {
+	if err := state.UpsertJobSummary(101, mock.JobSummary(alloc1.JobID)); err != nil {
 		t.Fatal(err)
 	}
-	err := state.UpsertAllocs(100,
-		[]*structs.Allocation{alloc1, alloc2})
+	err := state.UpsertAllocs(102, []*structs.Allocation{alloc1, alloc2})
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -421,7 +439,7 @@ func TestClient_WatchAllocs(t *testing.T) {
 	})
 
 	// Delete one allocation
-	err = state.DeleteEval(101, nil, []string{alloc1.ID})
+	err = state.DeleteEval(103, nil, []string{alloc1.ID})
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -432,8 +450,7 @@ func TestClient_WatchAllocs(t *testing.T) {
 	alloc2_2 := new(structs.Allocation)
 	*alloc2_2 = *alloc2
 	alloc2_2.DesiredStatus = structs.AllocDesiredStatusStop
-	err = state.UpsertAllocs(102,
-		[]*structs.Allocation{alloc2_2})
+	err = state.UpsertAllocs(104, []*structs.Allocation{alloc2_2})
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -459,6 +476,18 @@ func TestClient_WatchAllocs(t *testing.T) {
 	})
 }
 
+func waitTilNodeReady(client *Client, t *testing.T) {
+	testutil.WaitForResult(func() (bool, error) {
+		n := client.Node()
+		if n.Status != structs.NodeStatusReady {
+			return false, fmt.Errorf("node not registered")
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("err: %v", err)
+	})
+}
+
 func TestClient_SaveRestoreState(t *testing.T) {
 	ctestutil.ExecCompatible(t)
 	s1, _ := testServer(t, nil)
@@ -471,18 +500,27 @@ func TestClient_SaveRestoreState(t *testing.T) {
 	})
 	defer c1.Shutdown()
 
+	// Wait til the node is ready
+	waitTilNodeReady(c1, t)
+
 	// Create mock allocations
+	job := mock.Job()
 	alloc1 := mock.Alloc()
 	alloc1.NodeID = c1.Node().ID
+	alloc1.Job = job
+	alloc1.JobID = job.ID
 	task := alloc1.Job.TaskGroups[0].Tasks[0]
 	task.Config["command"] = "/bin/sleep"
-	task.Config["args"] = []string{"10"}
+	task.Config["args"] = []string{"100"}
 
 	state := s1.State()
-	if err := state.UpsertJobSummary(99, mock.JobSummary(alloc1.JobID)); err != nil {
+	if err := state.UpsertJob(100, job); err != nil {
 		t.Fatal(err)
 	}
-	if err := state.UpsertAllocs(100, []*structs.Allocation{alloc1}); err != nil {
+	if err := state.UpsertJobSummary(101, mock.JobSummary(alloc1.JobID)); err != nil {
+		t.Fatal(err)
+	}
+	if err := state.UpsertAllocs(102, []*structs.Allocation{alloc1}); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
@@ -491,7 +529,13 @@ func TestClient_SaveRestoreState(t *testing.T) {
 		c1.allocLock.RLock()
 		ar := c1.allocs[alloc1.ID]
 		c1.allocLock.RUnlock()
-		return ar != nil && ar.Alloc().ClientStatus == structs.AllocClientStatusRunning, nil
+		if ar == nil {
+			return false, fmt.Errorf("nil alloc runner")
+		}
+		if ar.Alloc().ClientStatus != structs.AllocClientStatusRunning {
+			return false, fmt.Errorf("client status: got %v; want %v", ar.Alloc().ClientStatus, structs.AllocClientStatusRunning)
+		}
+		return true, nil
 	}, func(err error) {
 		t.Fatalf("err: %v", err)
 	})

nomad/mock/mock.go

@@ -9,13 +9,14 @@ import (
 func Node() *structs.Node {
 	node := &structs.Node{
 		ID:         structs.GenerateUUID(),
+		SecretID:   structs.GenerateUUID(),
 		Datacenter: "dc1",
 		Name:       "foobar",
 		Attributes: map[string]string{
-			"kernel.name": "linux",
-			"arch":        "x86",
-			"version":     "0.1.0",
-			"driver.exec": "1",
+			"kernel.name":   "linux",
+			"arch":          "x86",
+			"nomad.version": "0.5.0",
+			"driver.exec":   "1",
 		},
 		Resources: &structs.Resources{
 			CPU:      4000,