Merge remote-tracking branch 'oss/master' into docs-updates

* oss/master: (55 commits)
  update dr replication docs with the promotion response (#3124)
  Make travis_wait Travis wait longer_wait
  changelog++
  Set allowed headers via API instead of defaulting to wildcard. (#3023)
  Fix formatting in mfa docs (#3122)
  Fix minor typo (#3120)
  Update go-plugin to include go-hclog support
  Unlock the statelock on unsuccessful sealInitCommon
  Remove a couple unneeded cancels
  Make seal/stepdown functions async internally so they can poke the request context
  Update mock-plugin (#3107)
  Fix minor grammatical error (#3110)
  docs: MFA API (#3109)
  Cut version 0.8.0-rc1
  Update version
  Migrate physical backends into separate packages (#3106)
  changeling ++
  changelog++
  changelog++
  credsutil: Include hyphen as part of reqStr (#3037)
  ...

.travis.yml

@@ -20,4 +20,5 @@ branches:
 
 script:
   - make bootstrap
-  - travis_wait 30 make test testrace
+  - travis_wait 30 make test
+  - travis_wait 30 make testrace

CHANGELOG.md

@@ -4,9 +4,11 @@ DEPRECATIONS/CHANGES:
 
  * Database Plugin Backends: Passwords generated for these backends now
    enforce stricter password requirements, as opposed to the previous behavior 
-   of returning a randomized UUID. Passwords are of length 20, and have a `A1a` 
+   of returning a randomized UUID. Passwords are of length 20, and have a `A1a-` 
    characters prepended to ensure stricter requirements. No regressions are 
-   expected from this change.
+   expected from this change. (For database backends that were previously
+   substituting underscores for hyphens in passwords, this will remain the
+   case.)
  * Lease Endpoints: The endpoints `sys/renew`, `sys/revoke`, `sys/revoke-prefix`,
    `sys/revoke-force` have been deprecated and relocated under `sys/leases`.
    Additionally, the deprecated path `sys/revoke-force` now requires the `sudo`
@@ -18,20 +20,31 @@ DEPRECATIONS/CHANGES:
 
 FEATURES:
 
+ * **Cassandra Storage**: Cassandra can now be used for Vault storage
  * **CockroachDB Storage**: CockroachDB can now be used for Vault storage
  * **CouchDB Storage**: CouchDB can now be used for Vault storage
  * **SAP HANA Database Plugin**: The `databases` backend can now manage users
    for SAP HANA databases
  * **Plugin Backends**: Vault now supports running secret and auth backends as
    plugins. Plugins can be mounted like normal backends and can be developed
    independently from Vault.
+ * **PROXY Protocol Support** Vault listeners can now be configured to honor
+   PROXY protocol v1 information to allow passing real client IPs into Vault. A
+   list of authorized addresses (IPs or subnets) can be defined and
+   accept/reject behavior controlled.
+ * **Lease lookup and browsing in the Vault Enterprise UI**: Vault Enterprise UI
+   now supports lookup and listing of leases and the associated actions from the 
+   `sys/leases` endpoints in the API. These are located in the new top level 
+   navigation item "Leases".
 
 IMPROVEMENTS:
 
  * api: Add client method for a secret renewer background process [GH-2886]
  * api: Add `RenewTokenAsSelf` [GH-2886]
  * api: Client timeout can now be adjusted with the `VAULT_CLIENT_TIMEOUT` env
    var or with a new API function [GH-2956]
+ * api/cli: Client will now attempt to look up SRV records for the given Vault
+   hostname [GH-3035]
  * audit/socket: Enhance reconnection logic and don't require the connection to
    be established at unseal time [GH-2934]
  * audit/file: Opportunistically try re-opening the file on error [GH-2999]
@@ -43,8 +56,13 @@ IMPROVEMENTS:
    token on stdout and does not store it via the token helper [GH-2855]
  * core: CORS allowed origins can now be configured [GH-2021]
  * core: Add metrics counters for audit log failures [GH-2863]
+ * cors: Allow setting allowed headers via the API instead of always using
+   wildcard [GH-3023]
  * secret/ssh: Allow specifying the key ID format using template values for CA
    type [GH-2888]
+ * server: Add `tls_client_ca_file` option for specifying a CA file to use for
+   client certificate verification when `tls_require_and_verify_client_cert` is
+   enabled [GH-3034]
  * storage/cockroachdb: Add CockroachDB storage backend [GH-2713]
  * storage/couchdb: Add CouchhDB storage backend [GH-2880]
  * storage/mssql: Add `max_parallel` [GH-3026]
@@ -53,6 +71,8 @@ IMPROVEMENTS:
  * storage/s3: More efficient paging when an object has a lot of subobjects
    [GH-2780]
  * sys/wrapping: Make `sys/wrapping/lookup` unauthenticated [GH-3084]
+ * sys/wrapping: Wrapped tokens now store the original request path of the data
+   [GH-3100]
  * telemetry: Add support for DogStatsD [GH-2490]
 
 BUG FIXES:

api/client.go

@@ -3,6 +3,7 @@ package api
 import (
 	"crypto/tls"
 	"fmt"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -358,12 +359,23 @@ func (c *Client) Clone() (*Client, error) {
 // configured for this client. This is an advanced method and generally
 // doesn't need to be called externally.
 func (c *Client) NewRequest(method, requestPath string) *Request {
+	// if SRV records exist (see https://tools.ietf.org/html/draft-andrews-http-srv-02), lookup the SRV
+	// record and take the highest match; this is not designed for high-availability, just discovery
+	var host string = c.addr.Host
+	if c.addr.Port() == "" {
+		// Internet Draft specifies that the SRV record is ignored if a port is given
+		_, addrs, err := net.LookupSRV("http", "tcp", c.addr.Hostname())
+		if err == nil && len(addrs) > 0 {
+			host = fmt.Sprintf("%s:%d", addrs[0].Target, addrs[0].Port)
+		}
+	}
+
 	req := &Request{
 		Method: method,
 		URL: &url.URL{
 			User:   c.addr.User,
 			Scheme: c.addr.Scheme,
-			Host:   c.addr.Host,
+			Host:   host,
 			Path:   path.Join(c.addr.Path, requestPath),
 		},
 		ClientToken: c.token,

api/secret.go

@@ -42,6 +42,7 @@ type SecretWrapInfo struct {
 	Token           string    `json:"token"`
 	TTL             int       `json:"ttl"`
 	CreationTime    time.Time `json:"creation_time"`
+	CreationPath    string    `json:"creation_path"`
 	WrappedAccessor string    `json:"wrapped_accessor"`
 }
 

api/sys_health.go

@@ -2,6 +2,11 @@ package api
 
 func (c *Sys) Health() (*HealthResponse, error) {
 	r := c.c.NewRequest("GET", "/v1/sys/health")
+	// If the code is 400 or above it will automatically turn into an error,
+	// but the sys/health API defaults to returning 5xx when not sealed or
+	// inited, so we force this code to be something else so we parse correctly
+	r.Params.Add("sealedcode", "299")
+	r.Params.Add("uninitcode", "299")
 	resp, err := c.c.RawRequest(r)
 	if err != nil {
 		return nil, err

audit/format.go

@@ -300,6 +300,7 @@ func (f *AuditFormatter) FormatResponse(
 			TTL:             int(resp.WrapInfo.TTL / time.Second),
 			Token:           token,
 			CreationTime:    resp.WrapInfo.CreationTime.Format(time.RFC3339Nano),
+			CreationPath:    resp.WrapInfo.CreationPath,
 			WrappedAccessor: resp.WrapInfo.WrappedAccessor,
 		}
 	}
@@ -406,6 +407,7 @@ type AuditResponseWrapInfo struct {
 	TTL             int    `json:"ttl"`
 	Token           string `json:"token"`
 	CreationTime    string `json:"creation_time"`
+	CreationPath    string `json:"creation_path"`
 	WrappedAccessor string `json:"wrapped_accessor,omitempty"`
 }
 

builtin/logical/mysql/secret_creds.go

@@ -66,6 +66,9 @@ func (b *backend) secretCredsRevoke(
 		return nil, fmt.Errorf("secret is missing username internal data")
 	}
 	username, ok := usernameRaw.(string)
+	if !ok {
+		return nil, fmt.Errorf("usernameRaw is not a string")
+	}
 
 	// Get our connection
 	db, err := b.DB(req.Storage)

builtin/logical/pki/path_issue_sign.go

@@ -277,6 +277,15 @@ func (b *backend) pathIssueSignCert(
 		}
 	}
 
+	if useCSR {
+		if role.UseCSRCommonName && data.Get("common_name").(string) != "" {
+			resp.AddWarning("the common_name field was provided but the role is set with \"use_csr_common_name\" set to true")
+		}
+		if role.UseCSRSANs && data.Get("alt_names").(string) != "" {
+			resp.AddWarning("the alt_names field was provided but the role is set with \"use_csr_sans\" set to true")
+		}
+	}
+
 	return resp, nil
 }
 

cli/commands.go

@@ -6,6 +6,7 @@ import (
 	auditFile "github.com/hashicorp/vault/builtin/audit/file"
 	auditSocket "github.com/hashicorp/vault/builtin/audit/socket"
 	auditSyslog "github.com/hashicorp/vault/builtin/audit/syslog"
+	"github.com/hashicorp/vault/physical"
 	"github.com/hashicorp/vault/version"
 
 	credAppId "github.com/hashicorp/vault/builtin/credential/app-id"
@@ -18,6 +19,23 @@ import (
 	credRadius "github.com/hashicorp/vault/builtin/credential/radius"
 	credUserpass "github.com/hashicorp/vault/builtin/credential/userpass"
 
+	physAzure "github.com/hashicorp/vault/physical/azure"
+	physCassandra "github.com/hashicorp/vault/physical/cassandra"
+	physCockroachDB "github.com/hashicorp/vault/physical/cockroachdb"
+	physConsul "github.com/hashicorp/vault/physical/consul"
+	physCouchDB "github.com/hashicorp/vault/physical/couchdb"
+	physDynamoDB "github.com/hashicorp/vault/physical/dynamodb"
+	physEtcd "github.com/hashicorp/vault/physical/etcd"
+	physFile "github.com/hashicorp/vault/physical/file"
+	physGCS "github.com/hashicorp/vault/physical/gcs"
+	physInmem "github.com/hashicorp/vault/physical/inmem"
+	physMSSQL "github.com/hashicorp/vault/physical/mssql"
+	physMySQL "github.com/hashicorp/vault/physical/mysql"
+	physPostgreSQL "github.com/hashicorp/vault/physical/postgresql"
+	physS3 "github.com/hashicorp/vault/physical/s3"
+	physSwift "github.com/hashicorp/vault/physical/swift"
+	physZooKeeper "github.com/hashicorp/vault/physical/zookeeper"
+
 	"github.com/hashicorp/vault/builtin/logical/aws"
 	"github.com/hashicorp/vault/builtin/logical/cassandra"
 	"github.com/hashicorp/vault/builtin/logical/consul"
@@ -63,7 +81,7 @@ func Commands(metaPtr *meta.Meta) map[string]cli.CommandFactory {
 			}, nil
 		},
 		"server": func() (cli.Command, error) {
-			return &command.ServerCommand{
+			c := &command.ServerCommand{
 				Meta: *metaPtr,
 				AuditBackends: map[string]audit.Factory{
 					"file":   auditFile.Factory,
@@ -98,9 +116,36 @@ func Commands(metaPtr *meta.Meta) map[string]cli.CommandFactory {
 					"totp":       totp.Factory,
 					"plugin":     plugin.Factory,
 				},
+
 				ShutdownCh: command.MakeShutdownCh(),
 				SighupCh:   command.MakeSighupCh(),
-			}, nil
+			}
+
+			c.PhysicalBackends = map[string]physical.Factory{
+				"azure":                  physAzure.NewAzureBackend,
+				"cassandra":              physCassandra.NewCassandraBackend,
+				"cockroachdb":            physCockroachDB.NewCockroachDBBackend,
+				"consul":                 physConsul.NewConsulBackend,
+				"couchdb":                physCouchDB.NewCouchDBBackend,
+				"couchdb_transactional":  physCouchDB.NewTransactionalCouchDBBackend,
+				"dynamodb":               physDynamoDB.NewDynamoDBBackend,
+				"etcd":                   physEtcd.NewEtcdBackend,
+				"file":                   physFile.NewFileBackend,
+				"file_transactional":     physFile.NewTransactionalFileBackend,
+				"gcs":                    physGCS.NewGCSBackend,
+				"inmem":                  physInmem.NewInmem,
+				"inmem_ha":               physInmem.NewInmemHA,
+				"inmem_transactional":    physInmem.NewTransactionalInmem,
+				"inmem_transactional_ha": physInmem.NewTransactionalInmemHA,
+				"mssql":                  physMSSQL.NewMSSQLBackend,
+				"mysql":                  physMySQL.NewMySQLBackend,
+				"postgresql":             physPostgreSQL.NewPostgreSQLBackend,
+				"s3":                     physS3.NewS3Backend,
+				"swift":                  physSwift.NewSwiftBackend,
+				"zookeeper":              physZooKeeper.NewZooKeeperBackend,
+			}
+
+			return c, nil
 		},
 
 		"ssh": func() (cli.Command, error) {

command/format.go

@@ -181,6 +181,7 @@ func (t TableFormatter) OutputSecret(ui cli.Ui, secret, s *api.Secret) error {
 		input = append(input, fmt.Sprintf("wrapping_token: %s %s", config.Delim, s.WrapInfo.Token))
 		input = append(input, fmt.Sprintf("wrapping_token_ttl: %s %s", config.Delim, (time.Second*time.Duration(s.WrapInfo.TTL)).String()))
 		input = append(input, fmt.Sprintf("wrapping_token_creation_time: %s %s", config.Delim, s.WrapInfo.CreationTime.String()))
+		input = append(input, fmt.Sprintf("wrapping_token_creation_path: %s %s", config.Delim, s.WrapInfo.CreationPath))
 		if s.WrapInfo.WrappedAccessor != "" {
 			input = append(input, fmt.Sprintf("wrapped_accessor: %s %s", config.Delim, s.WrapInfo.WrappedAccessor))
 		}

command/init.go

@@ -11,7 +11,7 @@ import (
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/helper/pgpkeys"
 	"github.com/hashicorp/vault/meta"
-	"github.com/hashicorp/vault/physical"
+	"github.com/hashicorp/vault/physical/consul"
 )
 
 // InitCommand is a Command that initializes a new Vault server.
@@ -36,7 +36,7 @@ func (c *InitCommand) Run(args []string) int {
 	flags.Var(&recoveryPgpKeys, "recovery-pgp-keys", "")
 	flags.BoolVar(&check, "check", false, "")
 	flags.BoolVar(&auto, "auto", false, "")
-	flags.StringVar(&consulServiceName, "consul-service", physical.DefaultServiceName, "")
+	flags.StringVar(&consulServiceName, "consul-service", consul.DefaultServiceName, "")
 	if err := flags.Parse(args); err != nil {
 		return 1
 	}