Support configurable certificate stores

* Issue #27: change certificates via API
* Issue #28: refactor listener config
* Issue #70: support Vault
* Issue #85: SNI support

.travis.yml

@@ -3,3 +3,13 @@ language: go
 go:
     - 1.6.2
     - 1.7beta2
+
+before_script:
+  - echo $HOSTNAME
+  - mkdir -p $GOPATH/bin
+  - wget https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip
+  - wget https://releases.hashicorp.com/vault/0.5.3/vault_0.5.3_linux_amd64.zip
+  - unzip -d $GOPATH/bin consul_0.6.4_linux_amd64.zip
+  - unzip -d $GOPATH/bin vault_0.5.3_linux_amd64.zip
+  - vault --version
+  - consul --version

Makefile

@@ -10,7 +10,7 @@ build:
 
 test:
 	$(GO) test -i ./...
-	$(GO) test ./...
+	$(GO) test -test.timeout 5s `go list ./... | grep -v '/vendor/'`
 
 gofmt:
 	gofmt -w `find . -type f -name '*.go' | grep -v vendor`

cert/consul_source.go

@@ -0,0 +1,124 @@
+package cert
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"log"
+	"net/url"
+	"path"
+	"reflect"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+)
+
+// ConsulSource implements a certificate source which loads
+// TLS and client authentication certificates from the consul KV store.
+// The CertURL/ClientCAURL must point to the base path of the certificates.
+// The TLS certificates are updated automatically when the KV store
+// changes.
+type ConsulSource struct {
+	CertURL     string
+	ClientCAURL string
+}
+
+const kvURLPrefix = "/v1/kv/"
+
+func parseConsulURL(consulURL, stripPrefix string) (client *api.Client, key string, err error) {
+	u, err := url.Parse(consulURL)
+	if err != nil {
+		return nil, "", err
+	}
+	var token string
+	if len(u.Query()["token"]) > 0 {
+		token = u.Query()["token"][0]
+	}
+	client, err = api.NewClient(&api.Config{Address: u.Host, Scheme: u.Scheme, Token: token})
+	if err != nil {
+		return nil, "", err
+	}
+	key = u.RequestURI()[len(stripPrefix):]
+	return client, key, nil
+}
+
+func (s ConsulSource) LoadClientCAs() (*x509.CertPool, error) {
+	if s.ClientCAURL == "" {
+		return nil, nil
+	}
+
+	client, key, err := parseConsulURL(s.ClientCAURL, kvURLPrefix)
+	if err != nil {
+		return nil, err
+	}
+
+	load := func(key string) (map[string][]byte, error) {
+		pemBlocks, _, err := getCerts(client, key, 0)
+		return pemBlocks, err
+	}
+	return newCertPool(key, load)
+}
+
+func (s ConsulSource) Certificates() chan []tls.Certificate {
+	if s.CertURL == "" {
+		return nil
+	}
+
+	client, key, err := parseConsulURL(s.CertURL, kvURLPrefix)
+	if err != nil {
+		log.Printf("[ERROR] cert: Failed to create consul client. %s", err)
+	}
+
+	pemBlocksCh := make(chan map[string][]byte, 1)
+	go watchKV(client, key, pemBlocksCh)
+
+	ch := make(chan []tls.Certificate, 1)
+	go func() {
+		for pemBlocks := range pemBlocksCh {
+			certs, err := loadCertificates(pemBlocks)
+			if err != nil {
+				log.Printf("[ERROR] cert: Failed to load certificates. %s", err)
+				continue
+			}
+			ch <- certs
+		}
+	}()
+	return ch
+}
+
+// watchKV monitors a key in the KV store for changes.
+func watchKV(client *api.Client, key string, pemBlocks chan map[string][]byte) {
+	var lastIndex uint64
+	var lastValue map[string][]byte
+
+	for {
+		value, index, err := getCerts(client, key, lastIndex)
+		if err != nil {
+			log.Printf("[WARN] cert: Error fetching certificates from %s. %v", key, err)
+			time.Sleep(time.Second)
+			continue
+		}
+
+		if !reflect.DeepEqual(value, lastValue) || index != lastIndex {
+			log.Printf("[INFO] cert: Certificate index changed to #%d", index)
+			pemBlocks <- value
+			lastValue, lastIndex = value, index
+		}
+	}
+}
+
+func getCerts(client *api.Client, key string, waitIndex uint64) (pemBlocks map[string][]byte, lastIndex uint64, err error) {
+	q := &api.QueryOptions{RequireConsistent: true, WaitIndex: waitIndex}
+	kvpairs, meta, err := client.KV().List(key, q)
+	if err != nil {
+		return nil, 0, fmt.Errorf("consul: list: %s", err)
+	}
+	if len(kvpairs) == 0 {
+		return nil, meta.LastIndex, nil
+	}
+	pemBlocks = map[string][]byte{}
+	for _, kvpair := range kvpairs {
+		pemBlocks[path.Base(kvpair.Key)] = kvpair.Value
+	}
+	return pemBlocks, meta.LastIndex, nil
+}

cert/file_source.go

@@ -0,0 +1,53 @@
+package cert
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+	"log"
+)
+
+// FileSource implements a certificate source for one
+// TLS and one client authentication certificate.
+// The certificates are loaded during startup and are cached
+// in memory until the program exits.
+// It exists to support the legacy configuration only. The
+// PathSource should be used instead.
+type FileSource struct {
+	CertFile       string
+	KeyFile        string
+	ClientAuthFile string
+}
+
+func (s FileSource) LoadClientCAs() (*x509.CertPool, error) {
+	return newCertPool(s.ClientAuthFile, func(path string) (map[string][]byte, error) {
+		if s.ClientAuthFile == "" {
+			return nil, nil
+		}
+		pemBlock, err := ioutil.ReadFile(path)
+		return map[string][]byte{path: pemBlock}, err
+	})
+}
+
+func (s FileSource) Certificates() chan []tls.Certificate {
+	ch := make(chan []tls.Certificate, 1)
+	ch <- []tls.Certificate{loadX509KeyPair(s.CertFile, s.KeyFile)}
+	close(ch)
+	return ch
+}
+
+func loadX509KeyPair(certFile, keyFile string) tls.Certificate {
+	if certFile == "" {
+		log.Fatalf("[FATAL] cert: CertFile is required")
+	}
+
+	if keyFile == "" {
+		keyFile = certFile
+	}
+
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Fatalf("[FATAL] cert: Error loading certificate. %s", err)
+	}
+	return cert
+}

cert/http_source.go

@@ -0,0 +1,31 @@
+package cert
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"time"
+)
+
+// HTTPSource implements a certificate source which loads
+// TLS and client authentication certificates from an HTTP/HTTPS server.
+// The CertURL/ClientCAURL must point to a text file in the directory
+// of the certificates. The text file contains all files that should
+// be loaded from this directory - one filename per line.
+// The TLS certificates are updated automatically when Refresh
+// is not zero. Refresh cannot be less than one second to prevent
+// busy loops.
+type HTTPSource struct {
+	CertURL     string
+	ClientCAURL string
+	Refresh     time.Duration
+}
+
+func (s HTTPSource) LoadClientCAs() (*x509.CertPool, error) {
+	return newCertPool(s.ClientCAURL, loadURL)
+}
+
+func (s HTTPSource) Certificates() chan []tls.Certificate {
+	ch := make(chan []tls.Certificate, 1)
+	go watch(ch, s.Refresh, s.CertURL, loadURL)
+	return ch
+}