Merge remote-tracking branch 'oss/master' into db-phy-cockroachdb

* oss/master: (161 commits)
  update gitignore
  changelog++
  Exclude /sys/leases/renew from registering with expiration manager (#2891)
  More cleanup
  Clarify/fix some configuration info.
  Add a convenience function for copying a client (#2887)
  Better error messages using ListObjects than using HeadBucket. Might be a bigger request but messages are better than BadRequest, how this changes effect the messages are in the issue (#2892)
  Add ACL info to Consul configuration page
  Return error on bad CORS and add Header specification to API request primitive
  Add Zyborg.Vault PowerShell module to libs list (#2869)
  changelog++
  CouchDB physical backend (#2880)
  Fix root paths test
  Add missing datadog vendored lib
  changelog++
  Fix up CORS.
  Cors headers (#2021)
  Address review feedback
  Fix the test error message
  Added utility on router to fetch mount entry using its ID
  ...

.gitignore

@@ -46,6 +46,7 @@ Vagrantfile
 
 .DS_Store
 .idea
+.vscode
 
 dist/*
 

.travis.yml

@@ -7,7 +7,7 @@ services:
   - docker
 
 go:
-  - 1.8.1
+  - 1.8.3
 
 matrix:
   allow_failures:

CHANGELOG.md

@@ -1,12 +1,111 @@
-## Next (Unreleased)
+## Next (unreleased)
+
+FEATURES:
+
+ * **CouchDB Storage**: CouchDB can now be used for Vault storage
+
+IMPROVEMENTS:
+
+ * core: CORS allowed origins can now be configured [GH-2021]
+ * storage/s3: More efficient paging when an object has a lot of subobjects
+   [GH-2780]
+ * telemetry: Add support for DogStatsD [GH-2490]
 
 BUG FIXES:
 
+ * api/health: Consider the response code from standby nodes [GH-2850]
+ * core: Relocated `sys/leases/renew` returns same payload as original 
+   `sys/leases` endpoint [GH-2891]
+
+## 0.7.3 (June 7th, 2017)
+
+SECURITY:
+
+ * Cert auth backend now checks validity of individual certificates: In
+   previous versions of Vault, validity (e.g. expiration) of individual leaf
+   certificates added for authentication was not checked. This was done to make
+   it easier for administrators to control lifecycles of individual
+   certificates added to the backend, e.g. the authentication material being
+   checked was access to that specific certificate's private key rather than
+   all private keys signed by a CA. However, this behavior is often unexpected
+   and as a result can lead to insecure deployments, so we are now validating
+   these certificates as well.
+ * App-ID path salting was skipped in 0.7.1/0.7.2: A regression in 0.7.1/0.7.2
+   caused the HMACing of any App-ID information stored in paths (including
+   actual app-IDs and user-IDs) to be unsalted and written as-is from the API.
+   In 0.7.3 any such paths will be automatically changed to salted versions on
+   access (e.g. login or read); however, if you created new app-IDs or user-IDs
+   in 0.7.1/0.7.2, you may want to consider whether any users with access to
+   Vault's underlying data store may have intercepted these values, and
+   revoke/roll them.
+
+DEPRECATIONS/CHANGES:
+
+ * Step-Down is Forwarded: When a step-down is issued against a non-active node
+   in an HA cluster, it will now forward the request to the active node.
+
+FEATURES:
+
+ * **ed25519 Signing/Verification in Transit with Key Derivation**: The
+   `transit` backend now supports generating
+   [ed25519](https://ed25519.cr.yp.to/) keys for signing and verification
+   functionality. These keys support derivation, allowing you to modify the
+   actual encryption key used by supplying a `context` value.
+ * **Key Version Specification for Encryption in Transit**: You can now specify
+   the version of a key you use to wish to generate a signature, ciphertext, or
+   HMAC. This can be controlled by the `min_encryption_version` key
+   configuration property.
+ * **Replication Primary Discovery (Enterprise)**: Replication primaries will
+   now advertise the addresses of their local HA cluster members to replication
+   secondaries. This helps recovery if the primary active node goes down and
+   neither service discovery nor load balancers are in use to steer clients.
+
+IMPROVEMENTS:
+
+ * api/health: Add Sys().Health() [GH-2805]
+ * audit: Add auth information to requests that error out [GH-2754]
+ * command/auth: Add `-no-store` option that prevents the auth command from
+   storing the returned token into the configured token helper [GH-2809]
+ * core/forwarding: Request forwarding now heartbeats to prevent unused
+   connections from being terminated by firewalls or proxies
+ * plugins/databases: Add MongoDB as an internal database plugin [GH-2698]
+ * storage/dynamodb: Add a method for checking the existence of children, 
+   speeding up deletion operations in the DynamoDB storage backend [GH-2722]
+ * storage/mysql: Add max_parallel parameter to MySQL backend [GH-2760]
+ * secret/databases: Support listing connections [GH-2823]
+ * secret/databases: Support custom renewal statements in Postgres database 
+   plugin [GH-2788]
+ * secret/databases: Use the role name as part of generated credentials
+   [GH-2812]
+ * ui (Enterprise): Transit key and secret browsing UI handle large lists better
+ * ui (Enterprise): root tokens are no longer persisted
+ * ui (Enterprise): support for mounting Database and TOTP secret backends
+
+BUG FIXES:
+
+ * auth/app-id: Fix regression causing loading of salts to be skipped
+ * auth/aws: Improve EC2 describe instances performance [GH-2766]
+ * auth/aws: Fix lookup of some instance profile ARNs [GH-2802]
+ * auth/aws: Resolve ARNs to internal AWS IDs which makes lookup at various
+   points (e.g. renewal time) more robust [GH-2814]
+ * auth/aws: Properly honor configured period when using IAM authentication
+   [GH-2825]
+ * auth/aws: Check that a bound IAM principal is not empty (in the current
+   state of the role) before requiring it match the previously authenticated
+   client [GH-2781]
+ * auth/cert: Fix panic on renewal [GH-2749]
+ * auth/cert: Certificate verification for non-CA certs [GH-2761]
+ * core/acl: Prevent race condition when compiling ACLs in some scenarios
+   [GH-2826]
  * secret/database: Increase wrapping token TTL; in a loaded scenario it could
    be too short
  * secret/generic: Allow integers to be set as the value of `ttl` field as the
    documentation claims is supported [GH-2699]
+ * secret/ssh: Added host key callback to ssh client config [GH-2752]
+ * storage/s3: Avoid a panic when some bad data is returned [GH-2785]
+ * storage/dynamodb: Fix list functions working improperly on Windows [GH-2789]
  * storage/file: Don't leak file descriptors in some error cases
+ * storage/swift: Fix pre-v3 project/tenant name reading [GH-2803]
 
 ## 0.7.2 (May 8th, 2017)
 
@@ -60,8 +159,10 @@ IMPROVEMENTS:
  * cli/revoke: Add `-self` option to allow revoking the currently active token
    [GH-2596]
  * core: Randomize x coordinate in Shamir shares [GH-2621]
- * tidy: Improvements to `auth/token/tidy` and `sys/leases/tidy` to handle more
-   cleanup cases [GH-2452]
+ * replication: Fix a bug when enabling `approle` on a primary before
+   secondaries were connected
+ * replication: Add heartbeating to ensure firewalls don't kill connections to
+   primaries
  * secret/pki: Add `no_store` option that allows certificates to be issued
    without being stored. This removes the ability to look up and/or add to a
    CRL but helps with scaling to very large numbers of certificates. [GH-2565]
@@ -78,6 +179,8 @@ IMPROVEMENTS:
    requests [GH-2466]
  * storage/s3: Use pooled transport for http client [GH-2481]
  * storage/swift: Allow domain values for V3 authentication [GH-2554]
+ * tidy: Improvements to `auth/token/tidy` and `sys/leases/tidy` to handle more
+   cleanup cases [GH-2452]
 
 BUG FIXES:
 

Makefile

@@ -1,44 +1,46 @@
 TEST?=$$(go list ./... | grep -v /vendor/)
 VETARGS?=-asmdecl -atomic -bool -buildtags -copylocks -methods -nilfunc -printf -rangeloops -shift -structtags -unsafeptr
 EXTERNAL_TOOLS=\
-	github.com/mitchellh/gox
+	github.com/mitchellh/gox \
+	github.com/kardianos/govendor
 BUILD_TAGS?=vault
+GOFMT_FILES?=$$(find . -name '*.go' | grep -v vendor)
 
 default: dev
 
 # bin generates the releaseable binaries for Vault
-bin: generate
+bin: fmtcheck generate
 	@CGO_ENABLED=0 BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/build.sh'"
 
 # dev creates binaries for testing Vault locally. These are put
 # into ./bin/ as well as $GOPATH/bin, except for quickdev which
 # is only put into /bin/
 quickdev: generate
 	@CGO_ENABLED=0 go build -i -tags='$(BUILD_TAGS)' -o bin/vault
-dev: generate
+dev: fmtcheck generate
 	@CGO_ENABLED=0 BUILD_TAGS='$(BUILD_TAGS)' VAULT_DEV_BUILD=1 sh -c "'$(CURDIR)/scripts/build.sh'"
 dev-dynamic: generate
 	@CGO_ENABLED=1 BUILD_TAGS='$(BUILD_TAGS)' VAULT_DEV_BUILD=1 sh -c "'$(CURDIR)/scripts/build.sh'"
 
 # test runs the unit tests and vets the code
-test: generate
+test: fmtcheck generate
 	CGO_ENABLED=0 VAULT_TOKEN= VAULT_ACC= go test -tags='$(BUILD_TAGS)' $(TEST) $(TESTARGS) -timeout=20m -parallel=4
 
-testcompile: generate
+testcompile: fmtcheck generate
 	@for pkg in $(TEST) ; do \
 		go test -v -c -tags='$(BUILD_TAGS)' $$pkg -parallel=4 ; \
 	done
 
 # testacc runs acceptance tests
-testacc: generate
+testacc: fmtcheck generate
 	@if [ "$(TEST)" = "./..." ]; then \
 		echo "ERROR: Set TEST to a specific package"; \
 		exit 1; \
 	fi
 	VAULT_ACC=1 go test -tags='$(BUILD_TAGS)' $(TEST) -v $(TESTARGS) -timeout 45m
 
 # testrace runs the race checker
-testrace: generate
+testrace: fmtcheck generate
 	CGO_ENABLED=1 VAULT_TOKEN= VAULT_ACC= go test -tags='$(BUILD_TAGS)' -race $(TEST) $(TESTARGS) -timeout=20m -parallel=4
 
 cover:
@@ -63,12 +65,19 @@ generate:
 # bootstrap the build by downloading additional tools
 bootstrap:
 	@for tool in  $(EXTERNAL_TOOLS) ; do \
-		echo "Installing $$tool" ; \
+		echo "Installing/Updating $$tool" ; \
 		go get -u $$tool; \
 	done
 
 proto:
 	protoc -I helper/forwarding -I vault -I ../../.. vault/*.proto --go_out=plugins=grpc:vault
 	protoc -I helper/forwarding -I vault -I ../../.. helper/forwarding/types.proto --go_out=plugins=grpc:helper/forwarding
 
-.PHONY: bin default generate test vet bootstrap
+fmtcheck:
+	@sh -c "'$(CURDIR)/scripts/gofmtcheck.sh'"
+
+fmt:
+	gofmt -w $(GOFMT_FILES)
+
+
+.PHONY: bin default generate test vet bootstrap fmt fmtcheck

api/client.go

@@ -6,11 +6,11 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
-	"path"
 
 	"golang.org/x/net/http2"
 
@@ -327,6 +327,11 @@ func (c *Client) ClearToken() {
 	c.token = ""
 }
 
+// Clone creates a copy of this client.
+func (c *Client) Clone() (*Client, error) {
+	return NewClient(c.config)
+}
+
 // NewRequest creates a new raw request object to query the Vault server
 // configured for this client. This is an advanced method and generally
 // doesn't need to be called externally.

api/request.go

@@ -14,6 +14,7 @@ type Request struct {
 	Method      string
 	URL         *url.URL
 	Params      url.Values
+	Headers     http.Header
 	ClientToken string
 	WrapTTL     string
 	Obj         interface{}
@@ -60,6 +61,14 @@ func (r *Request) ToHTTP() (*http.Request, error) {
 	req.URL.Host = r.URL.Host
 	req.Host = r.URL.Host
 
+	if r.Headers != nil {
+		for header, vals := range r.Headers {
+			for _, val := range vals {
+				req.Header.Add(header, val)
+			}
+		}
+	}
+
 	if len(r.ClientToken) != 0 {
 		req.Header.Set("X-Vault-Token", r.ClientToken)
 	}

api/response.go

@@ -25,8 +25,9 @@ func (r *Response) DecodeJSON(out interface{}) error {
 // this will fully consume the response body, but will not close it. The
 // body must still be closed manually.
 func (r *Response) Error() error {
-	// 200 to 399 are okay status codes
-	if r.StatusCode >= 200 && r.StatusCode < 400 {
+	// 200 to 399 are okay status codes. 429 is the code for health status of
+	// standby nodes.
+	if (r.StatusCode >= 200 && r.StatusCode < 400) || r.StatusCode == 429 {
 		return nil
 	}
 

api/sys_config_cors.go

@@ -0,0 +1,56 @@
+package api
+
+func (c *Sys) CORSStatus() (*CORSResponse, error) {
+	r := c.c.NewRequest("GET", "/v1/sys/config/cors")
+	resp, err := c.c.RawRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var result CORSResponse
+	err = resp.DecodeJSON(&result)
+	return &result, err
+}
+
+func (c *Sys) ConfigureCORS(req *CORSRequest) (*CORSResponse, error) {
+	r := c.c.NewRequest("PUT", "/v1/sys/config/cors")
+	if err := r.SetJSONBody(req); err != nil {
+		return nil, err
+	}
+
+	resp, err := c.c.RawRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var result CORSResponse
+	err = resp.DecodeJSON(&result)
+	return &result, err
+}
+
+func (c *Sys) DisableCORS() (*CORSResponse, error) {
+	r := c.c.NewRequest("DELETE", "/v1/sys/config/cors")
+
+	resp, err := c.c.RawRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var result CORSResponse
+	err = resp.DecodeJSON(&result)
+	return &result, err
+
+}
+
+type CORSRequest struct {
+	AllowedOrigins string `json:"allowed_origins"`
+	Enabled        bool   `json:"enabled"`
+}
+
+type CORSResponse struct {
+	AllowedOrigins string `json:"allowed_origins"`
+	Enabled        bool   `json:"enabled"`
+}

api/sys_health.go

@@ -0,0 +1,24 @@
+package api
+
+func (c *Sys) Health() (*HealthResponse, error) {
+	r := c.c.NewRequest("GET", "/v1/sys/health")
+	resp, err := c.c.RawRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var result HealthResponse
+	err = resp.DecodeJSON(&result)
+	return &result, err
+}
+
+type HealthResponse struct {
+	Initialized   bool   `json:"initialized"`
+	Sealed        bool   `json:"sealed"`
+	Standby       bool   `json:"standby"`
+	ServerTimeUTC int64  `json:"server_time_utc"`
+	Version       string `json:"version"`
+	ClusterName   string `json:"cluster_name,omitempty"`
+	ClusterID     string `json:"cluster_id,omitempty"`
+}

audit/audit.go

@@ -25,15 +25,21 @@ type Backend interface {
 	// GetHash is used to return the given data with the backend's hash,
 	// so that a caller can determine if a value in the audit log matches
 	// an expected plaintext value
-	GetHash(string) string
+	GetHash(string) (string, error)
 
 	// Reload is called on SIGHUP for supporting backends.
 	Reload() error
+
+	// Invalidate is called for path invalidation
+	Invalidate()
 }
 
 type BackendConfig struct {
-	// The salt that should be used for any secret obfuscation
-	Salt *salt.Salt
+	// The view to store the salt
+	SaltView logical.Storage
+
+	// The salt config that should be used for any secret obfuscation
+	SaltConfig *salt.Config
 
 	// Config is the opaque user configuration provided when mounting
 	Config map[string]string