From 6705ab8e09940974e0aa764cb037e4d60639e617 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 3 Oct 2016 11:27:12 -0700
Subject: [PATCH 01/10] Added the keygen command

---
 command/keygen.go | 45 +++++++++++++++++++++++++++++++++++++++++++++
 commands.go       |  5 +++++
 2 files changed, 50 insertions(+)
 create mode 100644 command/keygen.go

diff --git a/command/keygen.go b/command/keygen.go
new file mode 100644
index 000000000000..100c4a36df86
--- /dev/null
+++ b/command/keygen.go
@@ -0,0 +1,45 @@
+package command
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"strings"
+)
+
+// KeygenCommand is a Command implementation that generates an encryption
+// key for use in `nomad agent`.
+type KeygenCommand struct {
+	Meta
+}
+
+func (c *KeygenCommand) Run(_ []string) int {
+	key := make([]byte, 16)
+	n, err := rand.Reader.Read(key)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error reading random data: %s", err))
+		return 1
+	}
+	if n != 16 {
+		c.Ui.Error(fmt.Sprintf("Couldn't read enough entropy. Generate more entropy!"))
+		return 1
+	}
+
+	c.Ui.Output(base64.StdEncoding.EncodeToString(key))
+	return 0
+}
+
+func (c *KeygenCommand) Synopsis() string {
+	return "Generates a new encryption key"
+}
+
+func (c *KeygenCommand) Help() string {
+	helpText := `
+Usage: consul keygen
+
+  Generates a new encryption key that can be used to configure the
+  agent to encrypt traffic. The output of this command is already
+  in the proper format that the agent expects.
+`
+	return strings.TrimSpace(helpText)
+}
diff --git a/commands.go b/commands.go
index acef7f16c670..3af0689d9bfd 100644
--- a/commands.go
+++ b/commands.go
@@ -79,6 +79,11 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
+		"keygen": func() (cli.Command, error) {
+			return &command.KeygenCommand{
+				Meta: meta,
+			}, nil
+		},
 		"logs": func() (cli.Command, error) {
 			return &command.LogsCommand{
 				Meta: meta,

From 1bc0c02b54eb3350d3497989f60a16829da96550 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 3 Oct 2016 20:27:50 -0700
Subject: [PATCH 02/10] Added support for gossip encryption

---
 api/agent.go                                  |  55 +++++++
 command/agent/agent.go                        |  29 ++++
 command/agent/agent_endpoint.go               |  57 +++++++
 command/agent/config.go                       |  12 ++
 command/agent/config_parse.go                 |   1 +
 command/agent/http.go                         |   1 +
 command/agent/keyring.go                      | 106 +++++++++++++
 command/keyring.go                            | 148 ++++++++++++++++++
 nomad/server.go                               |   8 +
 nomad/structs/structs.go                      |  13 ++
 .../github.com/hashicorp/memberlist/config.go |   7 +
 .../hashicorp/memberlist/keyring.go           |  15 +-
 .../hashicorp/memberlist/memberlist.go        |   2 +-
 vendor/vendor.json                            |   8 +-
 14 files changed, 455 insertions(+), 7 deletions(-)
 create mode 100644 command/agent/keyring.go
 create mode 100644 command/keyring.go

diff --git a/api/agent.go b/api/agent.go
index 32c0c87d5b48..6795c43d1664 100644
--- a/api/agent.go
+++ b/api/agent.go
@@ -3,6 +3,8 @@ package api
 import (
 	"fmt"
 	"net/url"
+
+	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 // Agent encapsulates an API client which talks to Nomad's
@@ -16,6 +18,19 @@ type Agent struct {
 	region     string
 }
 
+// KeyringResponse is a unified key response and can be used for install,
+// remove, use, as well as listing key queries.
+type KeyringResponse struct {
+	Messages map[string]string
+	Keys     map[string]int
+	NumNodes int
+}
+
+// KeyringRequest is request objects for serf key operations.
+type KeyringRequest struct {
+	Key string
+}
+
 // Agent returns a new agent which can be used to query
 // the agent-specific endpoints.
 func (c *Client) Agent() *Agent {
@@ -157,6 +172,46 @@ func (a *Agent) SetServers(addrs []string) error {
 	return err
 }
 
+// ListKeys returns the list of installed keys
+func (a *Agent) ListKeys() (*KeyringResponse, error) {
+	var resp KeyringResponse
+	_, err := a.client.query("/v1/agent/keys/list", &resp, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+// InstallKey installs a key in the keyrings of all the serf members
+func (a *Agent) InstallKey(key string) (*KeyringResponse, error) {
+	args := structs.KeyringRequest{
+		Key: key,
+	}
+	var resp KeyringResponse
+	_, err := a.client.write("/v1/agent/keys/install", &args, &resp, nil)
+	return &resp, err
+}
+
+// UseKey uses a key from the keyring of serf members
+func (a *Agent) UseKey(key string) (*KeyringResponse, error) {
+	args := structs.KeyringRequest{
+		Key: key,
+	}
+	var resp KeyringResponse
+	_, err := a.client.write("/v1/agent/keys/use", &args, &resp, nil)
+	return &resp, err
+}
+
+// RemoveKey removes a particular key from keyrings of serf members
+func (a *Agent) RemoveKey(key string) (*KeyringResponse, error) {
+	args := structs.KeyringRequest{
+		Key: key,
+	}
+	var resp KeyringResponse
+	_, err := a.client.write("/v1/agent/keys/remove", &args, &resp, nil)
+	return &resp, err
+}
+
 // joinResponse is used to decode the response we get while
 // sending a member join request.
 type joinResponse struct {
diff --git a/command/agent/agent.go b/command/agent/agent.go
index 597c99bd8045..2c53bcc17332 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -5,6 +5,7 @@ import (
 	"io"
 	"log"
 	"net"
+	"os"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -371,6 +372,10 @@ func (a *Agent) setupServer() error {
 		return fmt.Errorf("server config setup failed: %s", err)
 	}
 
+	if err := a.setupKeyrings(conf); err != nil {
+		return fmt.Errorf("failed to configure keyring: %v", err)
+	}
+
 	// Create the server
 	server, err := nomad.NewServer(conf, a.consulSyncer, a.logger)
 	if err != nil {
@@ -431,6 +436,30 @@ func (a *Agent) setupServer() error {
 	return nil
 }
 
+// setupKeyrings is used to initialize and load keyrings during agent startup
+func (a *Agent) setupKeyrings(config *nomad.Config) error {
+	file := filepath.Join(a.config.DataDir, serfKeyring)
+
+	if a.config.Server.EncryptKey == "" {
+		goto LOAD
+	}
+	if _, err := os.Stat(file); err != nil {
+		if err := initKeyring(file, a.config.Server.EncryptKey); err != nil {
+			return err
+		}
+	}
+
+LOAD:
+	if _, err := os.Stat(file); err == nil {
+		config.SerfConfig.KeyringFile = file
+	}
+	if err := loadKeyringFile(config.SerfConfig); err != nil {
+		return err
+	}
+	// Success!
+	return nil
+}
+
 // setupClient is used to setup the client if enabled
 func (a *Agent) setupClient() error {
 	if !a.config.Client.Enabled {
diff --git a/command/agent/agent_endpoint.go b/command/agent/agent_endpoint.go
index 5e9a0c8d8fcf..96a3810577bd 100644
--- a/command/agent/agent_endpoint.go
+++ b/command/agent/agent_endpoint.go
@@ -3,7 +3,9 @@ package agent
 import (
 	"net"
 	"net/http"
+	"strings"
 
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/serf/serf"
 )
 
@@ -165,6 +167,61 @@ func (s *HTTPServer) updateServers(resp http.ResponseWriter, req *http.Request)
 	return nil, nil
 }
 
+// KeyringOperationRequest allows an operator to install/delete/use keys
+func (s *HTTPServer) KeyringOperationRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	s.logger.Printf("DIPTANU HERE")
+	srv := s.agent.Server()
+	if srv == nil {
+		return nil, CodedError(501, ErrInvalidMethod)
+	}
+
+	kmgr := srv.KeyManager()
+	var sresp *serf.KeyResponse
+	var err error
+
+	// Get the key from the req body
+	var args structs.KeyringRequest
+
+	//Get the op
+	op := strings.TrimPrefix(req.URL.Path, "/v1/agent/keys/")
+
+	switch op {
+	case "list":
+		sresp, err = kmgr.ListKeys()
+	case "install":
+		if err := s.getKey(req, &args); err != nil {
+			return nil, CodedError(500, err.Error())
+		}
+		sresp, err = kmgr.InstallKey(args.Key)
+	case "use":
+		if err := s.getKey(req, &args); err != nil {
+			return nil, CodedError(500, err.Error())
+		}
+		sresp, err = kmgr.UseKey(args.Key)
+	case "remove":
+		if err := s.getKey(req, &args); err != nil {
+			return nil, CodedError(500, err.Error())
+		}
+		sresp, err = kmgr.RemoveKey(args.Key)
+	default:
+		return nil, CodedError(404, "resource not found")
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	kresp := structs.KeyringResponse{
+		Messages: sresp.Messages,
+		Keys:     sresp.Keys,
+		NumNodes: sresp.NumNodes,
+	}
+	return kresp, nil
+}
+
+func (s *HTTPServer) getKey(req *http.Request, args *structs.KeyringRequest) error {
+	return decodeBody(req, args)
+}
+
 type agentSelf struct {
 	Config *Config                      `json:"config"`
 	Member Member                       `json:"member,omitempty"`
diff --git a/command/agent/config.go b/command/agent/config.go
index 27de7fff9208..38e20f97ef00 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io"
 	"net"
@@ -244,6 +245,14 @@ type ServerConfig struct {
 	// the cluster until an explicit join is received. If this is set to
 	// true, we ignore the leave, and rejoin the cluster on start.
 	RejoinAfterLeave bool `mapstructure:"rejoin_after_leave"`
+
+	// Encryption key to use for the Serf communication
+	EncryptKey string `mapstructure:"encrypt" json:"-"`
+}
+
+// EncryptBytes returns the encryption key configured.
+func (s *ServerConfig) EncryptBytes() ([]byte, error) {
+	return base64.StdEncoding.DecodeString(s.EncryptKey)
 }
 
 // Telemetry is the telemetry configuration for the server
@@ -666,6 +675,9 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 	if b.RejoinAfterLeave {
 		result.RejoinAfterLeave = true
 	}
+	if b.EncryptKey != "" {
+		result.EncryptKey = b.EncryptKey
+	}
 
 	// Add the schedulers
 	result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...)
diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go
index fbc3a2322986..78c13df61d61 100644
--- a/command/agent/config_parse.go
+++ b/command/agent/config_parse.go
@@ -484,6 +484,7 @@ func parseServer(result **ServerConfig, list *ast.ObjectList) error {
 		"retry_max",
 		"retry_interval",
 		"rejoin_after_leave",
+		"encrypt",
 	}
 	if err := checkHCLKeys(listVal, valid); err != nil {
 		return err
diff --git a/command/agent/http.go b/command/agent/http.go
index 73bb8d252d32..35b1585f4f99 100644
--- a/command/agent/http.go
+++ b/command/agent/http.go
@@ -122,6 +122,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
 	s.mux.HandleFunc("/v1/agent/members", s.wrap(s.AgentMembersRequest))
 	s.mux.HandleFunc("/v1/agent/force-leave", s.wrap(s.AgentForceLeaveRequest))
 	s.mux.HandleFunc("/v1/agent/servers", s.wrap(s.AgentServersRequest))
+	s.mux.HandleFunc("/v1/agent/keys/", s.wrap(s.KeyringOperationRequest))
 
 	s.mux.HandleFunc("/v1/regions", s.wrap(s.RegionListRequest))
 
diff --git a/command/agent/keyring.go b/command/agent/keyring.go
new file mode 100644
index 000000000000..fb9f262d03ef
--- /dev/null
+++ b/command/agent/keyring.go
@@ -0,0 +1,106 @@
+package agent
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/memberlist"
+	"github.com/hashicorp/serf/serf"
+)
+
+const (
+	serfKeyring = "serf.keyring"
+)
+
+// initKeyring will create a keyring file at a given path.
+func initKeyring(path, key string) error {
+	var keys []string
+
+	if keyBytes, err := base64.StdEncoding.DecodeString(key); err != nil {
+		return fmt.Errorf("Invalid key: %s", err)
+	} else if err := memberlist.ValidateKey(keyBytes); err != nil {
+		return fmt.Errorf("Invalid key: %s", err)
+	}
+
+	// Just exit if the file already exists.
+	if _, err := os.Stat(path); err == nil {
+		return nil
+	}
+
+	keys = append(keys, key)
+	keyringBytes, err := json.Marshal(keys)
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+		return err
+	}
+
+	fh, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+
+	if _, err := fh.Write(keyringBytes); err != nil {
+		os.Remove(path)
+		return err
+	}
+
+	return nil
+}
+
+// loadKeyringFile will load a gossip encryption keyring out of a file. The file
+// must be in JSON format and contain a list of encryption key strings.
+func loadKeyringFile(c *serf.Config) error {
+	if c.KeyringFile == "" {
+		return nil
+	}
+
+	if _, err := os.Stat(c.KeyringFile); err != nil {
+		return err
+	}
+
+	// Read in the keyring file data
+	keyringData, err := ioutil.ReadFile(c.KeyringFile)
+	if err != nil {
+		return err
+	}
+
+	// Decode keyring JSON
+	keys := make([]string, 0)
+	if err := json.Unmarshal(keyringData, &keys); err != nil {
+		return err
+	}
+
+	// Decode base64 values
+	keysDecoded := make([][]byte, len(keys))
+	for i, key := range keys {
+		keyBytes, err := base64.StdEncoding.DecodeString(key)
+		if err != nil {
+			return err
+		}
+		keysDecoded[i] = keyBytes
+	}
+
+	// Guard against empty keyring
+	if len(keysDecoded) == 0 {
+		return fmt.Errorf("no keys present in keyring file: %s", c.KeyringFile)
+	}
+
+	// Create the keyring
+	keyring, err := memberlist.NewKeyring(keysDecoded, keysDecoded[0])
+	if err != nil {
+		return err
+	}
+
+	c.MemberlistConfig.Keyring = keyring
+
+	// Success!
+	return nil
+}
diff --git a/command/keyring.go b/command/keyring.go
new file mode 100644
index 000000000000..28a01872118b
--- /dev/null
+++ b/command/keyring.go
@@ -0,0 +1,148 @@
+package command
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+
+	"github.com/mitchellh/cli"
+)
+
+// KeyringCommand is a Command implementation that handles querying, installing,
+// and removing gossip encryption keys from a keyring.
+type KeyringCommand struct {
+	Meta
+}
+
+func (c *KeyringCommand) Run(args []string) int {
+	var installKey, useKey, removeKey, token string
+	var listKeys bool
+
+	cmdFlags := flag.NewFlagSet("keys", flag.ContinueOnError)
+	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
+
+	cmdFlags.StringVar(&installKey, "install", "", "install key")
+	cmdFlags.StringVar(&useKey, "use", "", "use key")
+	cmdFlags.StringVar(&removeKey, "remove", "", "remove key")
+	cmdFlags.BoolVar(&listKeys, "list", false, "list keys")
+	cmdFlags.StringVar(&token, "token", "", "acl token")
+
+	if err := cmdFlags.Parse(args); err != nil {
+		return 1
+	}
+
+	c.Ui = &cli.PrefixedUi{
+		OutputPrefix: "",
+		InfoPrefix:   "==> ",
+		ErrorPrefix:  "",
+		Ui:           c.Ui,
+	}
+
+	// Only accept a single argument
+	found := listKeys
+	for _, arg := range []string{installKey, useKey, removeKey} {
+		if found && len(arg) > 0 {
+			c.Ui.Error("Only a single action is allowed")
+			return 1
+		}
+		found = found || len(arg) > 0
+	}
+
+	// Fail fast if no actionable args were passed
+	if !found {
+		c.Ui.Error(c.Help())
+		return 1
+	}
+
+	// All other operations will require a client connection
+	client, err := c.Meta.Client()
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error creating nomad cli client: %s", err))
+		return 1
+	}
+
+	if listKeys {
+		c.Ui.Info("Gathering installed encryption keys...")
+		r, err := client.Agent().ListKeys()
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		fmt.Printf("List %#v", r)
+		return 0
+	}
+
+	if installKey != "" {
+		c.Ui.Info("Installing new gossip encryption key...")
+		r, err := client.Agent().InstallKey(installKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		fmt.Printf("Install %#v", r)
+		return 0
+	}
+
+	if useKey != "" {
+		c.Ui.Info("Changing primary gossip encryption key...")
+		r, err := client.Agent().UseKey(useKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		fmt.Printf("Use %#v", r)
+		return 0
+	}
+
+	if removeKey != "" {
+		c.Ui.Info("Removing gossip encryption key...")
+		r, err := client.Agent().RemoveKey(removeKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		fmt.Printf("Remove %#v", r)
+		return 0
+	}
+
+	// Should never make it here
+	return 0
+}
+
+func (c *KeyringCommand) Help() string {
+	helpText := `
+Usage: nomad keyring [options]
+
+  Manages encryption keys used for gossip messages between nomad servers. Gossip
+  encryption is optional. When enabled, this command may be used to examine
+  active encryption keys in the cluster, add new keys, and remove old ones. When
+  combined, this functionality provides the ability to perform key rotation
+  cluster-wide, without disrupting the cluster.
+
+  All operations performed by this command can only be run against server nodes.
+
+  All variations of the keyring command return 0 if all nodes reply and there
+  are no errors. If any node fails to reply or reports failure, the exit code
+  will be 1.
+
+Options:
+
+  -install=<key>            Install a new encryption key. This will broadcast
+                            the new key to all members in the cluster.
+  -list                     List all keys currently in use within the cluster.
+  -remove=<key>             Remove the given key from the cluster. This
+                            operation may only be performed on keys which are
+                            not currently the primary key.
+  -token=""                 ACL token to use during requests. Defaults to that
+                            of the agent.
+  -use=<key>                Change the primary encryption key, which is used to
+                            encrypt messages. The key must already be installed
+                            before this operation can succeed.
+  -rpc-addr=127.0.0.1:8400  RPC address of the Consul agent.
+`
+	return strings.TrimSpace(helpText)
+}
+
+func (c *KeyringCommand) Synopsis() string {
+	return "Manages gossip layer encryption keys"
+}
diff --git a/nomad/server.go b/nomad/server.go
index 8d1851613dbd..b85ec3e9d5d5 100644
--- a/nomad/server.go
+++ b/nomad/server.go
@@ -935,3 +935,11 @@ func (s *Server) Stats() map[string]map[string]string {
 	}
 	return stats
 }
+
+func (s *Server) Region() string {
+	return s.config.Region
+}
+
+func (s *Server) Datacenter() string {
+	return s.config.Datacenter
+}
diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go
index fa40f84a94ec..e5fd3ae82156 100644
--- a/nomad/structs/structs.go
+++ b/nomad/structs/structs.go
@@ -3577,3 +3577,16 @@ func Encode(t MessageType, msg interface{}) ([]byte, error) {
 	err := codec.NewEncoder(&buf, MsgpackHandle).Encode(msg)
 	return buf.Bytes(), err
 }
+
+// KeyringResponse is a unified key response and can be used for install,
+// remove, use, as well as listing key queries.
+type KeyringResponse struct {
+	Messages map[string]string
+	Keys     map[string]int
+	NumNodes int
+}
+
+// KeyringRequest is request objects for serf key operations.
+type KeyringRequest struct {
+	Key string
+}
diff --git a/vendor/github.com/hashicorp/memberlist/config.go b/vendor/github.com/hashicorp/memberlist/config.go
index 85a93f4228d4..27a52ea56121 100644
--- a/vendor/github.com/hashicorp/memberlist/config.go
+++ b/vendor/github.com/hashicorp/memberlist/config.go
@@ -179,6 +179,11 @@ type Config struct {
 	// behavior for using LogOutput. You cannot specify both LogOutput and Logger
 	// at the same time.
 	Logger *log.Logger
+
+	// Size of Memberlist's internal channel which handles UDP messages. The
+	// size of this determines the size of the queue which Memberlist will keep
+	// while UDP messages are handled.
+	HandoffQueueDepth int
 }
 
 // DefaultLANConfig returns a sane set of configurations for Memberlist.
@@ -216,6 +221,8 @@ func DefaultLANConfig() *Config {
 		Keyring:   nil,
 
 		DNSConfigPath: "/etc/resolv.conf",
+
+		HandoffQueueDepth: 1024,
 	}
 }
 
diff --git a/vendor/github.com/hashicorp/memberlist/keyring.go b/vendor/github.com/hashicorp/memberlist/keyring.go
index be2201d48809..a2774a0ce085 100644
--- a/vendor/github.com/hashicorp/memberlist/keyring.go
+++ b/vendor/github.com/hashicorp/memberlist/keyring.go
@@ -58,6 +58,17 @@ func NewKeyring(keys [][]byte, primaryKey []byte) (*Keyring, error) {
 	return keyring, nil
 }
 
+// ValidateKey will check to see if the key is valid and returns an error if not.
+//
+// key should be either 16, 24, or 32 bytes to select AES-128,
+// AES-192, or AES-256.
+func ValidateKey(key []byte) error {
+	if l := len(key); l != 16 && l != 24 && l != 32 {
+		return fmt.Errorf("key size must be 16, 24 or 32 bytes")
+	}
+	return nil
+}
+
 // AddKey will install a new key on the ring. Adding a key to the ring will make
 // it available for use in decryption. If the key already exists on the ring,
 // this function will just return noop.
@@ -65,8 +76,8 @@ func NewKeyring(keys [][]byte, primaryKey []byte) (*Keyring, error) {
 // key should be either 16, 24, or 32 bytes to select AES-128,
 // AES-192, or AES-256.
 func (k *Keyring) AddKey(key []byte) error {
-	if l := len(key); l != 16 && l != 24 && l != 32 {
-		return fmt.Errorf("key size must be 16, 24 or 32 bytes")
+	if err := ValidateKey(key); err != nil {
+		return err
 	}
 
 	// No-op if key is already installed
diff --git a/vendor/github.com/hashicorp/memberlist/memberlist.go b/vendor/github.com/hashicorp/memberlist/memberlist.go
index 7d7e563ef0b8..6c8c93ba90be 100644
--- a/vendor/github.com/hashicorp/memberlist/memberlist.go
+++ b/vendor/github.com/hashicorp/memberlist/memberlist.go
@@ -129,7 +129,7 @@ func newMemberlist(conf *Config) (*Memberlist, error) {
 		leaveBroadcast: make(chan struct{}, 1),
 		udpListener:    udpLn,
 		tcpListener:    tcpLn,
-		handoff:        make(chan msgHandoff, 1024),
+		handoff:        make(chan msgHandoff, conf.HandoffQueueDepth),
 		nodeMap:        make(map[string]*nodeState),
 		nodeTimers:     make(map[string]*suspicion),
 		awareness:      newAwareness(conf.AwarenessMaxMultiplier),
diff --git a/vendor/vendor.json b/vendor/vendor.json
index f28df25d87e6..d43f1030c506 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -309,7 +309,7 @@
 			"revision": "02caa73df411debed164f520a6a1304778f8b88c",
 			"revisionTime": "2016-05-28T10:48:36Z"
 		},
-        {
+		{
 			"checksumSHA1": "BwY1agr5vzusgMT4bM7UeD3YVpw=",
 			"path": "github.com/docker/engine-api/types/mount",
 			"revision": "4290f40c056686fcaa5c9caf02eac1dde9315adf",
@@ -599,10 +599,10 @@
 			"revision": "0dc08b1671f34c4250ce212759ebd880f743d883"
 		},
 		{
-			"checksumSHA1": "8ytOx52G+38QMK4G194Kl6g6YGY=",
+			"checksumSHA1": "Ozk/S4U1x/OllNP2SsMYJjCl/gs=",
 			"path": "github.com/hashicorp/memberlist",
-			"revision": "b2053e314b4a87e5f0d2d47aeafd3e03be13da90",
-			"revisionTime": "2016-06-21T23:59:43Z"
+			"revision": "7ad712f5f34ec40aebe6ca47756d07898486a8d2",
+			"revisionTime": "2016-09-15T13:02:55Z"
 		},
 		{
 			"path": "github.com/hashicorp/net-rpc-msgpackrpc",

From 2c115a2519e878db51382da6a4e0b3e4811968de Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 3 Oct 2016 20:34:01 -0700
Subject: [PATCH 03/10] Changed the URL for keyring management

---
 api/agent.go                    |  8 ++++----
 command/agent/agent_endpoint.go |  3 +--
 command/agent/command.go        | 12 ++++++++++++
 command/agent/http.go           |  2 +-
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/api/agent.go b/api/agent.go
index 6795c43d1664..af06689d36ff 100644
--- a/api/agent.go
+++ b/api/agent.go
@@ -175,7 +175,7 @@ func (a *Agent) SetServers(addrs []string) error {
 // ListKeys returns the list of installed keys
 func (a *Agent) ListKeys() (*KeyringResponse, error) {
 	var resp KeyringResponse
-	_, err := a.client.query("/v1/agent/keys/list", &resp, nil)
+	_, err := a.client.query("/v1/agent/keyring/list", &resp, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -188,7 +188,7 @@ func (a *Agent) InstallKey(key string) (*KeyringResponse, error) {
 		Key: key,
 	}
 	var resp KeyringResponse
-	_, err := a.client.write("/v1/agent/keys/install", &args, &resp, nil)
+	_, err := a.client.write("/v1/agent/keyring/install", &args, &resp, nil)
 	return &resp, err
 }
 
@@ -198,7 +198,7 @@ func (a *Agent) UseKey(key string) (*KeyringResponse, error) {
 		Key: key,
 	}
 	var resp KeyringResponse
-	_, err := a.client.write("/v1/agent/keys/use", &args, &resp, nil)
+	_, err := a.client.write("/v1/agent/keyring/use", &args, &resp, nil)
 	return &resp, err
 }
 
@@ -208,7 +208,7 @@ func (a *Agent) RemoveKey(key string) (*KeyringResponse, error) {
 		Key: key,
 	}
 	var resp KeyringResponse
-	_, err := a.client.write("/v1/agent/keys/remove", &args, &resp, nil)
+	_, err := a.client.write("/v1/agent/keyring/remove", &args, &resp, nil)
 	return &resp, err
 }
 
diff --git a/command/agent/agent_endpoint.go b/command/agent/agent_endpoint.go
index 96a3810577bd..34b7f7ac7424 100644
--- a/command/agent/agent_endpoint.go
+++ b/command/agent/agent_endpoint.go
@@ -169,7 +169,6 @@ func (s *HTTPServer) updateServers(resp http.ResponseWriter, req *http.Request)
 
 // KeyringOperationRequest allows an operator to install/delete/use keys
 func (s *HTTPServer) KeyringOperationRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
-	s.logger.Printf("DIPTANU HERE")
 	srv := s.agent.Server()
 	if srv == nil {
 		return nil, CodedError(501, ErrInvalidMethod)
@@ -183,7 +182,7 @@ func (s *HTTPServer) KeyringOperationRequest(resp http.ResponseWriter, req *http
 	var args structs.KeyringRequest
 
 	//Get the op
-	op := strings.TrimPrefix(req.URL.Path, "/v1/agent/keys/")
+	op := strings.TrimPrefix(req.URL.Path, "/v1/agent/keyring/")
 
 	switch op {
 	case "list":
diff --git a/command/agent/command.go b/command/agent/command.go
index 8e999c60a73c..cc8ec3caa124 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -83,6 +83,7 @@ func (c *Command) readConfig() *Config {
 	flags.Var((*sliceflag.StringFlag)(&cmdConfig.Server.RetryJoin), "retry-join", "")
 	flags.IntVar(&cmdConfig.Server.RetryMaxAttempts, "retry-max", 0, "")
 	flags.StringVar(&cmdConfig.Server.RetryInterval, "retry-interval", "", "")
+	flags.StringVar(&cmdConfig.Server.EncryptKey, "encrypt", "", "gossip encryption key")
 
 	// Client-only options
 	flags.StringVar(&cmdConfig.Client.StateDir, "state-dir", "", "")
@@ -189,6 +190,17 @@ func (c *Command) readConfig() *Config {
 		return config
 	}
 
+	if config.Server.EncryptKey != "" {
+		if _, err := config.Server.EncryptBytes(); err != nil {
+			c.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err))
+			return nil
+		}
+		keyfile := filepath.Join(config.DataDir, serfKeyring)
+		if _, err := os.Stat(keyfile); err == nil {
+			c.Ui.Error("WARNING: keyring exists but -encrypt given, using keyring")
+		}
+	}
+
 	// Parse the RetryInterval.
 	dur, err := time.ParseDuration(config.Server.RetryInterval)
 	if err != nil {
diff --git a/command/agent/http.go b/command/agent/http.go
index 35b1585f4f99..ec2c4fdb47a1 100644
--- a/command/agent/http.go
+++ b/command/agent/http.go
@@ -122,7 +122,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
 	s.mux.HandleFunc("/v1/agent/members", s.wrap(s.AgentMembersRequest))
 	s.mux.HandleFunc("/v1/agent/force-leave", s.wrap(s.AgentForceLeaveRequest))
 	s.mux.HandleFunc("/v1/agent/servers", s.wrap(s.AgentServersRequest))
-	s.mux.HandleFunc("/v1/agent/keys/", s.wrap(s.KeyringOperationRequest))
+	s.mux.HandleFunc("/v1/agent/keyring/", s.wrap(s.KeyringOperationRequest))
 
 	s.mux.HandleFunc("/v1/regions", s.wrap(s.RegionListRequest))
 

From 7a5362d5c15f6f3e85f1659ef7b77fbd92ca5510 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 3 Oct 2016 21:01:41 -0700
Subject: [PATCH 04/10] Fixed the cli

---
 command/keyring.go | 23 ++++++++++++++++-------
 commands.go        |  5 +++++
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/command/keyring.go b/command/keyring.go
index 28a01872118b..05489728dc10 100644
--- a/command/keyring.go
+++ b/command/keyring.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/hashicorp/nomad/api"
 	"github.com/mitchellh/cli"
 )
 
@@ -68,7 +69,7 @@ func (c *KeyringCommand) Run(args []string) int {
 			c.Ui.Error(fmt.Sprintf("error: %s", err))
 			return 1
 		}
-		fmt.Printf("List %#v", r)
+		c.handleKeyResponse(r)
 		return 0
 	}
 
@@ -79,7 +80,7 @@ func (c *KeyringCommand) Run(args []string) int {
 			c.Ui.Error(fmt.Sprintf("error: %s", err))
 			return 1
 		}
-		fmt.Printf("Install %#v", r)
+		c.handleKeyResponse(r)
 		return 0
 	}
 
@@ -90,7 +91,7 @@ func (c *KeyringCommand) Run(args []string) int {
 			c.Ui.Error(fmt.Sprintf("error: %s", err))
 			return 1
 		}
-		fmt.Printf("Use %#v", r)
+		c.handleKeyResponse(r)
 		return 0
 	}
 
@@ -101,7 +102,7 @@ func (c *KeyringCommand) Run(args []string) int {
 			c.Ui.Error(fmt.Sprintf("error: %s", err))
 			return 1
 		}
-		fmt.Printf("Remove %#v", r)
+		c.handleKeyResponse(r)
 		return 0
 	}
 
@@ -109,6 +110,17 @@ func (c *KeyringCommand) Run(args []string) int {
 	return 0
 }
 
+func (c *KeyringCommand) handleKeyResponse(resp *api.KeyringResponse) {
+	out := make([]string, len(resp.Keys)+1)
+	out[0] = "Key"
+	i := 1
+	for k := range resp.Keys {
+		out[i] = fmt.Sprintf("%s", k)
+		i = i + 1
+	}
+	c.Ui.Output(formatList(out))
+}
+
 func (c *KeyringCommand) Help() string {
 	helpText := `
 Usage: nomad keyring [options]
@@ -133,12 +145,9 @@ Options:
   -remove=<key>             Remove the given key from the cluster. This
                             operation may only be performed on keys which are
                             not currently the primary key.
-  -token=""                 ACL token to use during requests. Defaults to that
-                            of the agent.
   -use=<key>                Change the primary encryption key, which is used to
                             encrypt messages. The key must already be installed
                             before this operation can succeed.
-  -rpc-addr=127.0.0.1:8400  RPC address of the Consul agent.
 `
 	return strings.TrimSpace(helpText)
 }
diff --git a/commands.go b/commands.go
index 3af0689d9bfd..36f8e92c5f7c 100644
--- a/commands.go
+++ b/commands.go
@@ -84,6 +84,11 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
+		"keyring": func() (cli.Command, error) {
+			return &command.KeyringCommand{
+				Meta: meta,
+			}, nil
+		},
 		"logs": func() (cli.Command, error) {
 			return &command.LogsCommand{
 				Meta: meta,

From 1bc1b9b364bd375724952f3c841dc9ff0c80569e Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Tue, 4 Oct 2016 08:01:10 -0700
Subject: [PATCH 05/10] Added some tests

---
 command/agent/agent.go                       |  1 +
 command/agent/agent_endpoint.go              | 10 +--
 command/agent/config-test-fixtures/basic.hcl |  1 +
 command/agent/config_parse_test.go           |  1 +
 command/agent/keyring.go                     |  2 +-
 command/agent/keyring_test.go                | 85 ++++++++++++++++++++
 command/agent/server/serf.keyring            |  1 +
 command/keygen_test.go                       | 27 +++++++
 nomad/server.go                              |  7 ++
 9 files changed, 127 insertions(+), 8 deletions(-)
 create mode 100644 command/agent/keyring_test.go
 create mode 100644 command/agent/server/serf.keyring
 create mode 100644 command/keygen_test.go

diff --git a/command/agent/agent.go b/command/agent/agent.go
index 2c53bcc17332..71e298d27211 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -372,6 +372,7 @@ func (a *Agent) setupServer() error {
 		return fmt.Errorf("server config setup failed: %s", err)
 	}
 
+	// Sets up the keyring for gossip encryption
 	if err := a.setupKeyrings(conf); err != nil {
 		return fmt.Errorf("failed to configure keyring: %v", err)
 	}
diff --git a/command/agent/agent_endpoint.go b/command/agent/agent_endpoint.go
index 34b7f7ac7424..e22afcd4f36c 100644
--- a/command/agent/agent_endpoint.go
+++ b/command/agent/agent_endpoint.go
@@ -188,17 +188,17 @@ func (s *HTTPServer) KeyringOperationRequest(resp http.ResponseWriter, req *http
 	case "list":
 		sresp, err = kmgr.ListKeys()
 	case "install":
-		if err := s.getKey(req, &args); err != nil {
+		if err := decodeBody(req, &args); err != nil {
 			return nil, CodedError(500, err.Error())
 		}
 		sresp, err = kmgr.InstallKey(args.Key)
 	case "use":
-		if err := s.getKey(req, &args); err != nil {
+		if err := decodeBody(req, &args); err != nil {
 			return nil, CodedError(500, err.Error())
 		}
 		sresp, err = kmgr.UseKey(args.Key)
 	case "remove":
-		if err := s.getKey(req, &args); err != nil {
+		if err := decodeBody(req, &args); err != nil {
 			return nil, CodedError(500, err.Error())
 		}
 		sresp, err = kmgr.RemoveKey(args.Key)
@@ -217,10 +217,6 @@ func (s *HTTPServer) KeyringOperationRequest(resp http.ResponseWriter, req *http
 	return kresp, nil
 }
 
-func (s *HTTPServer) getKey(req *http.Request, args *structs.KeyringRequest) error {
-	return decodeBody(req, args)
-}
-
 type agentSelf struct {
 	Config *Config                      `json:"config"`
 	Member Member                       `json:"member,omitempty"`
diff --git a/command/agent/config-test-fixtures/basic.hcl b/command/agent/config-test-fixtures/basic.hcl
index 62a2992a2cca..96f9e6eb1411 100644
--- a/command/agent/config-test-fixtures/basic.hcl
+++ b/command/agent/config-test-fixtures/basic.hcl
@@ -68,6 +68,7 @@ server {
 	retry_max = 3
 	retry_interval = "15s"
 	rejoin_after_leave = true
+    encrypt = "abc"
 }
 telemetry {
 	statsite_address = "127.0.0.1:1234"
diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go
index 1809fffee94b..e6f05a38796f 100644
--- a/command/agent/config_parse_test.go
+++ b/command/agent/config_parse_test.go
@@ -85,6 +85,7 @@ func TestConfig_Parse(t *testing.T) {
 					RetryInterval:     "15s",
 					RejoinAfterLeave:  true,
 					RetryMaxAttempts:  3,
+					EncryptKey:        "abc",
 				},
 				Telemetry: &Telemetry{
 					StatsiteAddr:             "127.0.0.1:1234",
diff --git a/command/agent/keyring.go b/command/agent/keyring.go
index fb9f262d03ef..ca509506b29d 100644
--- a/command/agent/keyring.go
+++ b/command/agent/keyring.go
@@ -13,7 +13,7 @@ import (
 )
 
 const (
-	serfKeyring = "serf.keyring"
+	serfKeyring = "server/serf.keyring"
 )
 
 // initKeyring will create a keyring file at a given path.
diff --git a/command/agent/keyring_test.go b/command/agent/keyring_test.go
new file mode 100644
index 000000000000..3e6d64ce410a
--- /dev/null
+++ b/command/agent/keyring_test.go
@@ -0,0 +1,85 @@
+package agent
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestAgent_LoadKeyrings(t *testing.T) {
+	key := "tbLJg26ZJyJ9pK3qhc9jig=="
+
+	// Should be no configured keyring file by default
+	dir1, agent1 := makeAgent(t, nil)
+	defer os.RemoveAll(dir1)
+	defer agent1.Shutdown()
+
+	c := agent1.server.GetConfig()
+	if c.SerfConfig.KeyringFile != "" {
+		t.Fatalf("bad: %#v", c.SerfConfig.KeyringFile)
+	}
+	if c.SerfConfig.MemberlistConfig.Keyring != nil {
+		t.Fatalf("keyring should not be loaded")
+	}
+
+	// Server should auto-load LAN and WAN keyring files
+	dir2, agent2 := makeAgent(t, func(c *Config) {
+		file := filepath.Join(c.DataDir, serfKeyring)
+		if err := initKeyring(file, key); err != nil {
+			t.Fatalf("err: %s", err)
+		}
+	})
+	defer os.RemoveAll(dir2)
+	defer agent2.Shutdown()
+
+	c = agent2.server.GetConfig()
+	if c.SerfConfig.KeyringFile == "" {
+		t.Fatalf("should have keyring file")
+	}
+	if c.SerfConfig.MemberlistConfig.Keyring == nil {
+		t.Fatalf("keyring should be loaded")
+	}
+}
+
+func TestAgent_InitKeyring(t *testing.T) {
+	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
+	key2 := "4leC33rgtXKIVUr9Nr0snQ=="
+	expected := fmt.Sprintf(`["%s"]`, key1)
+
+	dir, err := ioutil.TempDir("", "consul")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.RemoveAll(dir)
+
+	file := filepath.Join(dir, "keyring")
+
+	// First initialize the keyring
+	if err := initKeyring(file, key1); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if string(content) != expected {
+		t.Fatalf("bad: %s", content)
+	}
+
+	// Try initializing again with a different key
+	if err := initKeyring(file, key2); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Content should still be the same
+	content, err = ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if string(content) != expected {
+		t.Fatalf("bad: %s", content)
+	}
+}
diff --git a/command/agent/server/serf.keyring b/command/agent/server/serf.keyring
new file mode 100644
index 000000000000..b27c67887b58
--- /dev/null
+++ b/command/agent/server/serf.keyring
@@ -0,0 +1 @@
+["tbLJg26ZJyJ9pK3qhc9jig=="]
\ No newline at end of file
diff --git a/command/keygen_test.go b/command/keygen_test.go
new file mode 100644
index 000000000000..726dd841f948
--- /dev/null
+++ b/command/keygen_test.go
@@ -0,0 +1,27 @@
+package command
+
+import (
+	"encoding/base64"
+	"testing"
+
+	"github.com/mitchellh/cli"
+)
+
+func TestKeygenCommand(t *testing.T) {
+	ui := new(cli.MockUi)
+	c := &KeygenCommand{Meta: Meta{Ui: ui}}
+	code := c.Run(nil)
+	if code != 0 {
+		t.Fatalf("bad: %d", code)
+	}
+
+	output := ui.OutputWriter.String()
+	result, err := base64.StdEncoding.DecodeString(output)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if len(result) != 16 {
+		t.Fatalf("bad: %#v", result)
+	}
+}
diff --git a/nomad/server.go b/nomad/server.go
index b85ec3e9d5d5..35e1fd94eb75 100644
--- a/nomad/server.go
+++ b/nomad/server.go
@@ -936,10 +936,17 @@ func (s *Server) Stats() map[string]map[string]string {
 	return stats
 }
 
+// Region retuns the region of the server
 func (s *Server) Region() string {
 	return s.config.Region
 }
 
+// Datacenter returns the data center of the server
 func (s *Server) Datacenter() string {
 	return s.config.Datacenter
 }
+
+// GetConfig returns the config of the server for testing purposes only
+func (s *Server) GetConfig() *Config {
+	return s.config
+}

From b2698045c7adf43132b529bbd9907c0281cdcd0e Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Tue, 4 Oct 2016 09:12:13 -0700
Subject: [PATCH 06/10] Added tests for keyring operations

---
 command/agent/agent_endpoint_test.go | 65 ++++++++++++++++++++++++++++
 command/agent/server/serf.keyring    |  1 -
 2 files changed, 65 insertions(+), 1 deletion(-)
 delete mode 100644 command/agent/server/serf.keyring

diff --git a/command/agent/agent_endpoint_test.go b/command/agent/agent_endpoint_test.go
index c79f9dae509c..4132f63798e0 100644
--- a/command/agent/agent_endpoint_test.go
+++ b/command/agent/agent_endpoint_test.go
@@ -1,11 +1,15 @@
 package agent
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 func TestHTTP_AgentSelf(t *testing.T) {
@@ -177,3 +181,64 @@ func TestHTTP_AgentSetServers(t *testing.T) {
 		}
 	})
 }
+
+func TestHTTP_AgentListKeys(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+
+	httpTest(t, func(c *Config) {
+		c.Server.EncryptKey = key1
+	}, func(s *TestServer) {
+		req, err := http.NewRequest("GET", "/v1/agent/keyring/list", nil)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW := httptest.NewRecorder()
+
+		out, err := s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		kresp := out.(structs.KeyringResponse)
+		if len(kresp.Keys) != 1 {
+			t.Fatalf("bad: %v", kresp)
+		}
+	})
+}
+
+func TestHTTP_AgentInstallKey(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+	key2 := "wH1Bn9hlJ0emgWB1JttVRA=="
+
+	httpTest(t, func(c *Config) {
+		c.Server.EncryptKey = key1
+	}, func(s *TestServer) {
+		b, err := json.Marshal(&structs.KeyringRequest{Key: key2})
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW := httptest.NewRecorder()
+
+		_, err = s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		req, err = http.NewRequest("GET", "/v1/agent/keyring/list", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW = httptest.NewRecorder()
+
+		out, err := s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		kresp := out.(structs.KeyringResponse)
+		if len(kresp.Keys) != 2 {
+			t.Fatalf("bad: %v", kresp)
+		}
+	})
+}
diff --git a/command/agent/server/serf.keyring b/command/agent/server/serf.keyring
deleted file mode 100644
index b27c67887b58..000000000000
--- a/command/agent/server/serf.keyring
+++ /dev/null
@@ -1 +0,0 @@
-["tbLJg26ZJyJ9pK3qhc9jig=="]
\ No newline at end of file

From e3dea06f8132a2d15af200499596f05d08c68141 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Tue, 4 Oct 2016 09:32:44 -0700
Subject: [PATCH 07/10] Added a test for removal of keys

---
 command/agent/agent_endpoint_test.go | 47 ++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/command/agent/agent_endpoint_test.go b/command/agent/agent_endpoint_test.go
index 4132f63798e0..a7e28861d1f3 100644
--- a/command/agent/agent_endpoint_test.go
+++ b/command/agent/agent_endpoint_test.go
@@ -242,3 +242,50 @@ func TestHTTP_AgentInstallKey(t *testing.T) {
 		}
 	})
 }
+
+func TestHTTP_AgentRemoveKey(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+	key2 := "wH1Bn9hlJ0emgWB1JttVRA=="
+
+	httpTest(t, func(c *Config) {
+		c.Server.EncryptKey = key1
+	}, func(s *TestServer) {
+		b, err := json.Marshal(&structs.KeyringRequest{Key: key2})
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW := httptest.NewRecorder()
+		_, err = s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+
+		req, err = http.NewRequest("GET", "/v1/agent/keyring/remove", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW = httptest.NewRecorder()
+		if _, err = s.Server.KeyringOperationRequest(respW, req); err != nil {
+			t.Fatalf("err: %s", err)
+		}
+
+		req, err = http.NewRequest("GET", "/v1/agent/keyring/list", nil)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW = httptest.NewRecorder()
+		out, err := s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		kresp := out.(structs.KeyringResponse)
+		if len(kresp.Keys) != 1 {
+			t.Fatalf("bad: %v", kresp)
+		}
+	})
+}

From 8514abef4e28757f117f21f9afade308625134e9 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Tue, 4 Oct 2016 23:30:01 -0700
Subject: [PATCH 08/10] Added some docs

---
 command/agent/keyring_test.go                 |  2 +-
 command/keygen.go                             |  2 +-
 command/keyring.go                            |  9 +--
 website/source/docs/agent/config.html.md      | 10 ++++
 .../source/docs/commands/keygen.html.md.erb   | 30 ++++++++++
 .../source/docs/commands/keyring.html.md.erb  | 58 +++++++++++++++++++
 6 files changed, 103 insertions(+), 8 deletions(-)
 create mode 100644 website/source/docs/commands/keygen.html.md.erb
 create mode 100644 website/source/docs/commands/keyring.html.md.erb

diff --git a/command/agent/keyring_test.go b/command/agent/keyring_test.go
index 3e6d64ce410a..475dc622354c 100644
--- a/command/agent/keyring_test.go
+++ b/command/agent/keyring_test.go
@@ -48,7 +48,7 @@ func TestAgent_InitKeyring(t *testing.T) {
 	key2 := "4leC33rgtXKIVUr9Nr0snQ=="
 	expected := fmt.Sprintf(`["%s"]`, key1)
 
-	dir, err := ioutil.TempDir("", "consul")
+	dir, err := ioutil.TempDir("", "nomad")
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
diff --git a/command/keygen.go b/command/keygen.go
index 100c4a36df86..c5aed9f63355 100644
--- a/command/keygen.go
+++ b/command/keygen.go
@@ -35,7 +35,7 @@ func (c *KeygenCommand) Synopsis() string {
 
 func (c *KeygenCommand) Help() string {
 	helpText := `
-Usage: consul keygen
+Usage: nomad keygen
 
   Generates a new encryption key that can be used to configure the
   agent to encrypt traffic. The output of this command is already
diff --git a/command/keyring.go b/command/keyring.go
index 05489728dc10..b8ea5c6b0cac 100644
--- a/command/keyring.go
+++ b/command/keyring.go
@@ -75,34 +75,31 @@ func (c *KeyringCommand) Run(args []string) int {
 
 	if installKey != "" {
 		c.Ui.Info("Installing new gossip encryption key...")
-		r, err := client.Agent().InstallKey(installKey)
+		_, err := client.Agent().InstallKey(installKey)
 		if err != nil {
 			c.Ui.Error(fmt.Sprintf("error: %s", err))
 			return 1
 		}
-		c.handleKeyResponse(r)
 		return 0
 	}
 
 	if useKey != "" {
 		c.Ui.Info("Changing primary gossip encryption key...")
-		r, err := client.Agent().UseKey(useKey)
+		_, err := client.Agent().UseKey(useKey)
 		if err != nil {
 			c.Ui.Error(fmt.Sprintf("error: %s", err))
 			return 1
 		}
-		c.handleKeyResponse(r)
 		return 0
 	}
 
 	if removeKey != "" {
 		c.Ui.Info("Removing gossip encryption key...")
-		r, err := client.Agent().RemoveKey(removeKey)
+		_, err := client.Agent().RemoveKey(removeKey)
 		if err != nil {
 			c.Ui.Error(fmt.Sprintf("error: %s", err))
 			return 1
 		}
-		c.handleKeyResponse(r)
 		return 0
 	}
 
diff --git a/website/source/docs/agent/config.html.md b/website/source/docs/agent/config.html.md
index dacc727b64f9..414cf425f689 100644
--- a/website/source/docs/agent/config.html.md
+++ b/website/source/docs/agent/config.html.md
@@ -391,6 +391,16 @@ configured on client nodes.
     join any nodes when it starts up. Addresses can be given as an IP, a domain
     name, or an IP:Port pair. If the port isn't specified the default Serf port,
     4648, is used.  DNS names may also be used.
+  * <a id="encrypt">`encrypt`</a> Specifies the secret key to use for encryption
+    of Nomad server's gossip network traffic. This key must be 16-bytes that are
+    Base64-encoded. The easiest way to create an encryption key is to use nomad
+    keygen. All the servers within a cluster must share the same encryption key
+    to communicate. The provided key is automatically persisted to the data
+    directory and loaded automatically whenever the agent is restarted. This
+    means that to encrypt Nomad server's gossip protocol, this option only needs
+    to be provided once on each agent's initial startup sequence. If it is
+    provided after Nomad has been initialized with an encryption key, then the
+    provided key is ignored and a warning will be displayed.
 
 ## Client-specific Options
 
diff --git a/website/source/docs/commands/keygen.html.md.erb b/website/source/docs/commands/keygen.html.md.erb
new file mode 100644
index 000000000000..d92a79ce7122
--- /dev/null
+++ b/website/source/docs/commands/keygen.html.md.erb
@@ -0,0 +1,30 @@
+---
+layout: "docs"
+page_title: "Commands: keygen"
+sidebar_current: "docs-commands-keygen"
+description: >
+  The `keygen` command generates an encryption key that can be used for Nomad
+  server's gossip traffic encryption. The keygen command uses a
+  cryptographically strong pseudo-random number generator to generate the key.
+---
+
+
+# Command: `keygen`
+
+The `keygen` command generates an encryption key that can be used for Nomad
+server's gossip traffic encryption.  The keygen command uses a cryptographically
+strong pseudo-random number generator to generate the key.
+
+## Usage 
+
+```
+nomad keygen
+```
+
+## Example
+
+```
+nomad keygen
+YgZOXLMhC7TtZqeghMT8+w==
+```
+
diff --git a/website/source/docs/commands/keyring.html.md.erb b/website/source/docs/commands/keyring.html.md.erb
new file mode 100644
index 000000000000..d0f38d22aff7
--- /dev/null
+++ b/website/source/docs/commands/keyring.html.md.erb
@@ -0,0 +1,58 @@
+---
+layout: "docs"
+page_title: "Commands: keyring"
+sidebar_current: "docs-commands-keyring"
+---
+
+# Command: `keyring`
+
+The `keyring` command is used to examine and modify the encryption keys used in
+Nomad server. It is capable of
+distributing new encryption keys to the cluster, retiring old encryption keys,
+and changing the keys used by the cluster to encrypt messages.
+
+Nomad allows multiple encryption keys to be in use simultaneously. This is
+intended to provide a transition state while the cluster converges. It is the
+responsibility of the operator to ensure that only the required encryption keys
+are installed on the cluster. You can review the installed keys using the
+`-list` argument, and remove unneeded keys with `-remove`.
+
+All operations performed by this command can only be run against server nodes
+and will effect the entire cluster.
+
+All variations of the `keyring` command return 0 if all nodes reply and there
+are no errors. If any node fails to reply or reports failure, the exit code
+will be 1.
+
+
+## Usage
+
+Usage: `nomad keyring [options]`
+
+Only one actionable argument may be specified per run, including `-list`,
+`-install`, `-remove`, and `-use`.
+
+The list of available flags are:
+
+* `-list` - List all keys currently in use within the cluster.
+
+* `-install` - Install a new encryption key. This will broadcast the new key to
+  all members in the cluster.
+
+* `-use` - Change the primary encryption key, which is used to encrypt messages.
+  The key must already be installed before this operation can succeed.
+
+* `-remove` - Remove the given key from the cluster. This operation may only be
+  performed on keys which are not currently the primary key.
+
+## Output
+
+The output of the `nomad keyring -list` command consolidates information from
+all the Nomad servers from all datacenters and regions to provide a simple and
+easy to understand view of the cluster. 
+
+```
+==> Gathering installed encryption keys...
+Key
+PGm64/neoebUBqYR/lZTbA==
+```

From 35a02c08bc6a6eb034dd165a2608e5d70dbac26a Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 6 Oct 2016 10:35:05 +0530
Subject: [PATCH 09/10] Fixed some docs

---
 api/agent.go                                     | 8 +++-----
 command/keyring.go                               | 2 +-
 website/source/docs/commands/keyring.html.md.erb | 6 +++---
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/api/agent.go b/api/agent.go
index af06689d36ff..054d0c343a57 100644
--- a/api/agent.go
+++ b/api/agent.go
@@ -3,8 +3,6 @@ package api
 import (
 	"fmt"
 	"net/url"
-
-	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 // Agent encapsulates an API client which talks to Nomad's
@@ -184,7 +182,7 @@ func (a *Agent) ListKeys() (*KeyringResponse, error) {
 
 // InstallKey installs a key in the keyrings of all the serf members
 func (a *Agent) InstallKey(key string) (*KeyringResponse, error) {
-	args := structs.KeyringRequest{
+	args := KeyringRequest{
 		Key: key,
 	}
 	var resp KeyringResponse
@@ -194,7 +192,7 @@ func (a *Agent) InstallKey(key string) (*KeyringResponse, error) {
 
 // UseKey uses a key from the keyring of serf members
 func (a *Agent) UseKey(key string) (*KeyringResponse, error) {
-	args := structs.KeyringRequest{
+	args := KeyringRequest{
 		Key: key,
 	}
 	var resp KeyringResponse
@@ -204,7 +202,7 @@ func (a *Agent) UseKey(key string) (*KeyringResponse, error) {
 
 // RemoveKey removes a particular key from keyrings of serf members
 func (a *Agent) RemoveKey(key string) (*KeyringResponse, error) {
-	args := structs.KeyringRequest{
+	args := KeyringRequest{
 		Key: key,
 	}
 	var resp KeyringResponse
diff --git a/command/keyring.go b/command/keyring.go
index b8ea5c6b0cac..a3e0f724dd4e 100644
--- a/command/keyring.go
+++ b/command/keyring.go
@@ -122,7 +122,7 @@ func (c *KeyringCommand) Help() string {
 	helpText := `
 Usage: nomad keyring [options]
 
-  Manages encryption keys used for gossip messages between nomad servers. Gossip
+  Manages encryption keys used for gossip messages between Nomad servers. Gossip
   encryption is optional. When enabled, this command may be used to examine
   active encryption keys in the cluster, add new keys, and remove old ones. When
   combined, this functionality provides the ability to perform key rotation
diff --git a/website/source/docs/commands/keyring.html.md.erb b/website/source/docs/commands/keyring.html.md.erb
index d0f38d22aff7..46e45c4d7c5f 100644
--- a/website/source/docs/commands/keyring.html.md.erb
+++ b/website/source/docs/commands/keyring.html.md.erb
@@ -7,9 +7,9 @@ sidebar_current: "docs-commands-keyring"
 # Command: `keyring`
 
 The `keyring` command is used to examine and modify the encryption keys used in
-Nomad server. It is capable of
-distributing new encryption keys to the cluster, retiring old encryption keys,
-and changing the keys used by the cluster to encrypt messages.
+Nomad server. It is capable of distributing new encryption keys to the cluster,
+retiring old encryption keys, and changing the keys used by the cluster to
+encrypt messages.
 
 Nomad allows multiple encryption keys to be in use simultaneously. This is
 intended to provide a transition state while the cluster converges. It is the

From aa7c8c22f9d22094f6fb45cf3084730e8cfde662 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 17 Oct 2016 10:44:50 -0700
Subject: [PATCH 10/10] Added general options

---
 command/agent/command.go |  3 +++
 command/keyring.go       | 23 +++++++++++++----------
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/command/agent/command.go b/command/agent/command.go
index cc8ec3caa124..2d68a19ef597 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -819,6 +819,9 @@ Server Options:
     bootstrapping the cluster. Once <num> servers have joined eachother,
     Nomad initiates the bootstrap process.
 
+  -encrypt=<key>
+    Provides the gossip encryption key
+
   -join=<address>
     Address of an agent to join at start time. Can be specified
     multiple times.
diff --git a/command/keyring.go b/command/keyring.go
index a3e0f724dd4e..7c74f5eaa5cc 100644
--- a/command/keyring.go
+++ b/command/keyring.go
@@ -1,7 +1,6 @@
 package command
 
 import (
-	"flag"
 	"fmt"
 	"strings"
 
@@ -19,16 +18,16 @@ func (c *KeyringCommand) Run(args []string) int {
 	var installKey, useKey, removeKey, token string
 	var listKeys bool
 
-	cmdFlags := flag.NewFlagSet("keys", flag.ContinueOnError)
-	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
+	flags := c.Meta.FlagSet("keys", FlagSetClient)
+	flags.Usage = func() { c.Ui.Output(c.Help()) }
 
-	cmdFlags.StringVar(&installKey, "install", "", "install key")
-	cmdFlags.StringVar(&useKey, "use", "", "use key")
-	cmdFlags.StringVar(&removeKey, "remove", "", "remove key")
-	cmdFlags.BoolVar(&listKeys, "list", false, "list keys")
-	cmdFlags.StringVar(&token, "token", "", "acl token")
+	flags.StringVar(&installKey, "install", "", "install key")
+	flags.StringVar(&useKey, "use", "", "use key")
+	flags.StringVar(&removeKey, "remove", "", "remove key")
+	flags.BoolVar(&listKeys, "list", false, "list keys")
+	flags.StringVar(&token, "token", "", "acl token")
 
-	if err := cmdFlags.Parse(args); err != nil {
+	if err := flags.Parse(args); err != nil {
 		return 1
 	}
 
@@ -134,7 +133,11 @@ Usage: nomad keyring [options]
   are no errors. If any node fails to reply or reports failure, the exit code
   will be 1.
 
-Options:
+General Options:
+
+  ` + generalOptionsUsage() + `
+
+Keyring Options:
 
   -install=<key>            Install a new encryption key. This will broadcast
                             the new key to all members in the cluster.