From 326af9b578ab1a2282f147ea7aa2e78fbd6de4c1 Mon Sep 17 00:00:00 2001 From: Raphael Randschau Date: Tue, 13 Jun 2017 14:34:31 +0100 Subject: [PATCH 1/4] feat(storage): add couchdb backend --- physical/couchdb.go | 237 ++++++++++++++++++ physical/couchdb_test.go | 112 +++++++++ physical/physical.go | 1 + .../configuration/storage/couchdb.html.md | 39 +++ 4 files changed, 389 insertions(+) create mode 100644 physical/couchdb.go create mode 100644 physical/couchdb_test.go create mode 100644 website/source/docs/configuration/storage/couchdb.html.md diff --git a/physical/couchdb.go b/physical/couchdb.go new file mode 100644 index 000000000000..f06171660cb4 --- /dev/null +++ b/physical/couchdb.go @@ -0,0 +1,237 @@ +package physical + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/armon/go-metrics" + log "github.com/mgutz/logxi/v1" +) + +// CouchDBBackend allows the management of couchdb users +type CouchDBBackend struct { + logger log.Logger + client *couchDBClient +} + +type couchDBClient struct { + endpoint string + username string + password string + *http.Client +} + +type couchDBListItem struct { + ID string `json:"id"` + Key string `json:"key"` + Value struct { + Revision string + } `json:"value"` +} + +type couchDBList struct { + TotalRows int `json:"total_rows"` + Offset int `json:"offset"` + Rows []couchDBListItem `json:"rows"` +} + +func (m *couchDBClient) rev(key string) (string, error) { + req, err := http.NewRequest("HEAD", fmt.Sprintf("%s/%s", m.endpoint, key), nil) + if err != nil { + return "", err + } + req.SetBasicAuth(m.username, m.password) + + resp, err := m.Client.Do(req) + if err != nil { + return "", err + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", nil + } + etag := resp.Header.Get("Etag") + if len(etag) < 2 { + return "", nil + } + return etag[1 : len(etag)-1], nil +} + +func (m *couchDBClient) put(e couchDBEntry) error { + bs, err := json.Marshal(e) + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", fmt.Sprintf("%s/%s", m.endpoint, e.ID), bytes.NewReader(bs)) + if err != nil { + return err + } + req.SetBasicAuth(m.username, m.password) + _, err = m.Client.Do(req) + + return err +} + +func (m *couchDBClient) get(key string) (*Entry, error) { + req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", m.endpoint, url.PathEscape(key)), nil) + if err != nil { + return nil, err + } + req.SetBasicAuth(m.username, m.password) + resp, err := m.Client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } else if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET returned %s", resp.Status) + } + bs, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + entry := couchDBEntry{} + if err := json.Unmarshal(bs, &entry); err != nil { + return nil, err + } + return entry.Entry, nil +} + +func (m *couchDBClient) list(prefix string) ([]couchDBListItem, error) { + req, _ := http.NewRequest("GET", fmt.Sprintf("%s/_all_docs", m.endpoint), nil) + req.SetBasicAuth(m.username, m.password) + values := req.URL.Query() + values.Set("skip", "0") + values.Set("limit", "100") + values.Set("include_docs", "false") + if prefix != "" { + values.Set("startkey", fmt.Sprintf("%q", prefix)) + values.Set("endkey", fmt.Sprintf("%q", prefix+"{}")) + } + req.URL.RawQuery = values.Encode() + + resp, err := m.Client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + results := couchDBList{} + if err := json.Unmarshal(data, &results); err != nil { + return nil, err + } + + return results.Rows, nil +} + +func newCouchDBBackend(conf map[string]string, logger log.Logger) (Backend, error) { + endpoint := os.Getenv("COUCHDB_ENDPOINT") + if endpoint == "" { + endpoint = conf["endpoint"] + } + if endpoint == "" { + return nil, fmt.Errorf("missing endpoint") + } + + username := os.Getenv("COUCHDB_USERNAME") + if username == "" { + username = conf["username"] + } + + password := os.Getenv("COUCHDB_PASSWORD") + if password == "" { + password = conf["password"] + } + + return &CouchDBBackend{ + client: &couchDBClient{ + endpoint: endpoint, + username: username, + password: password, + Client: &http.Client{}, + }, + logger: logger, + }, nil +} + +type couchDBEntry struct { + Entry *Entry `json:"entry"` + Rev string `json:"_rev,omitempty"` + ID string `json:"_id"` + Deleted *bool `json:"_deleted,omitempty"` +} + +// Put is used to insert or update an entry +func (m *CouchDBBackend) Put(entry *Entry) error { + defer metrics.MeasureSince([]string{"couchdb", "put"}, time.Now()) + + revision, _ := m.client.rev(url.PathEscape(entry.Key)) + + return m.client.put(couchDBEntry{ + Entry: entry, + Rev: revision, + ID: url.PathEscape(entry.Key), + }) +} + +// Get is used to fetch an entry +func (m *CouchDBBackend) Get(key string) (*Entry, error) { + defer metrics.MeasureSince([]string{"couchdb", "get"}, time.Now()) + + return m.client.get(key) +} + +// Delete is used to permanently delete an entry +func (m *CouchDBBackend) Delete(key string) error { + defer metrics.MeasureSince([]string{"couchdb", "delete"}, time.Now()) + + revision, _ := m.client.rev(url.PathEscape(key)) + deleted := true + return m.client.put(couchDBEntry{ + ID: url.PathEscape(key), + Rev: revision, + Deleted: &deleted, + }) +} + +// List is used to list all the keys under a given prefix +func (m *CouchDBBackend) List(prefix string) ([]string, error) { + defer metrics.MeasureSince([]string{"couchdb", "list"}, time.Now()) + + items, err := m.client.list(prefix) + if err != nil { + return nil, err + } + + var out []string + seen := make(map[string]interface{}) + for _, result := range items { + trimmed := strings.TrimPrefix(result.ID, prefix) + sep := strings.Index(trimmed, "/") + if sep == -1 { + out = append(out, trimmed) + } else { + trimmed = trimmed[:sep+1] + if _, ok := seen[trimmed]; !ok { + out = append(out, trimmed) + seen[trimmed] = struct{}{} + } + } + } + return out, nil +} diff --git a/physical/couchdb_test.go b/physical/couchdb_test.go new file mode 100644 index 000000000000..132edcb2fcbb --- /dev/null +++ b/physical/couchdb_test.go @@ -0,0 +1,112 @@ +package physical + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/hashicorp/vault/helper/logformat" + log "github.com/mgutz/logxi/v1" + dockertest "gopkg.in/ory-am/dockertest.v3" +) + +func TestCouchDBBackend(t *testing.T) { + cleanup, endpoint, username, password := prepareCouchdbDBTestContainer(t) + defer cleanup() + + logger := logformat.NewVaultLogger(log.LevelTrace) + + b, err := NewBackend("couchdb", logger, map[string]string{ + "endpoint": endpoint, + "username": username, + "password": password, + }) + if err != nil { + t.Fatalf("err: %s", err) + } + + testBackend(t, b) + testBackend_ListPrefix(t, b) +} + +func prepareCouchdbDBTestContainer(t *testing.T) (cleanup func(), retAddress, username, password string) { + // If environment variable is set, assume caller wants to target a real + // DynamoDB. + if os.Getenv("COUCHDB_ENDPOINT") != "" { + return func() {}, os.Getenv("COUCHDB_ENDPOINT"), os.Getenv("COUCHDB_USERNAME"), os.Getenv("COUCHDB_PASSWORD") + } + + pool, err := dockertest.NewPool("") + if err != nil { + t.Fatalf("Failed to connect to docker: %s", err) + } + + resource, err := pool.Run("couchdb", "1.6", []string{}) + if err != nil { + t.Fatalf("Could not start local DynamoDB: %s", err) + } + + retAddress = "http://localhost:" + resource.GetPort("5984/tcp") + cleanup = func() { + err := pool.Purge(resource) + if err != nil { + t.Fatalf("Failed to cleanup local DynamoDB: %s", err) + } + } + + // exponential backoff-retry, because the couchDB may not be able to accept + // connections yet + if err := pool.Retry(func() error { + var err error + resp, err := http.Get(retAddress) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("Expected couchdb to return status code 200, got (%s) instead.", resp.Status) + } + return nil + }); err != nil { + t.Fatalf("Could not connect to docker: %s", err) + } + + dbName := fmt.Sprintf("vault-test-%d", time.Now().Unix()) + { + req, err := http.NewRequest("PUT", fmt.Sprintf("%s/%s", retAddress, dbName), nil) + if err != nil { + t.Fatalf("Could not create create database request: %q", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Could not create database: %q", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + bs, _ := ioutil.ReadAll(resp.Body) + t.Fatalf("Failed to create database: %s %s\n", resp.Status, string(bs)) + } + } + { + req, err := http.NewRequest("PUT", fmt.Sprintf("%s/_config/admins/admin", retAddress), strings.NewReader(`"admin"`)) + if err != nil { + t.Fatalf("Could not create admin user request: %q", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Could not create admin user: %q", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + bs, _ := ioutil.ReadAll(resp.Body) + t.Fatalf("Failed to create admin user: %s %s\n", resp.Status, string(bs)) + } + } + + return cleanup, retAddress + "/" + dbName, "admin", "admin" +} diff --git a/physical/physical.go b/physical/physical.go index b35d281ceba0..4f9d42d748af 100644 --- a/physical/physical.go +++ b/physical/physical.go @@ -151,6 +151,7 @@ var builtinBackends = map[string]Factory{ "mssql": newMsSQLBackend, "mysql": newMySQLBackend, "postgresql": newPostgreSQLBackend, + "couchdb": newCouchDBBackend, "swift": newSwiftBackend, "gcs": newGCSBackend, } diff --git a/website/source/docs/configuration/storage/couchdb.html.md b/website/source/docs/configuration/storage/couchdb.html.md new file mode 100644 index 000000000000..edd756418d73 --- /dev/null +++ b/website/source/docs/configuration/storage/couchdb.html.md @@ -0,0 +1,39 @@ +--- +layout: "docs" +page_title: "CouchDB - Storage Backends - Configuration" +sidebar_current: "docs-configuration-storage-couchdb" +description: |- + The CouchDB storage backend is used to persist Vault's data in a CouchDB + database. +--- + +# CouchDB Storage Backend + +The CouchDB storage backend is used to persist Vault's data in +[CouchDB][couchdb] table. + +- **Community Supported** – the CouchDB storage backend is supported by the + community. While it has undergone review by HashiCorp employees, they may not + be as knowledgeable about the technology. If you encounter problems with them, + you may be referred to the original author. + +```hcl +storage "couchdb" { + endpoint = "https://my-couchdb-dns.tld:5984/my-database" + username = "admin" + password = "admin" +} +``` + +## `couchdb` Parameters + +- `endpoint` `(string: "")` – Specifies your CouchDB endpoint. This can also be provided via the + environment variable `COUCHDB_ENDPOINT`. + +- `username` `(string: "")` – Specifies the user to authenticate as. This can also be provided via the + environment variable `COUCHDB_USERNAME`. + +- `password` `(string: "")` – Specifies the user to authenticate as. This can also be provided via the + environment variable `COUCHDB_PASSWORD`. + +[couchdb]: http://couchdb.apache.org/ From a5dcaf1869f8d1f4d354226ff2272f174b692bc0 Mon Sep 17 00:00:00 2001 From: Raphael Randschau Date: Fri, 16 Jun 2017 21:39:01 +0200 Subject: [PATCH 2/4] chore(couchdb): add pseudotransactional support --- physical/couchdb.go | 56 +++++++++++++++++-- physical/couchdb_test.go | 19 +++++++ physical/physical.go | 29 +++++----- .../configuration/storage/couchdb.html.md | 3 + 4 files changed, 89 insertions(+), 18 deletions(-) diff --git a/physical/couchdb.go b/physical/couchdb.go index f06171660cb4..ff08bec33b2d 100644 --- a/physical/couchdb.go +++ b/physical/couchdb.go @@ -17,8 +17,9 @@ import ( // CouchDBBackend allows the management of couchdb users type CouchDBBackend struct { - logger log.Logger - client *couchDBClient + logger log.Logger + client *couchDBClient + permitPool *PermitPool } type couchDBClient struct { @@ -139,7 +140,7 @@ func (m *couchDBClient) list(prefix string) ([]couchDBListItem, error) { return results.Rows, nil } -func newCouchDBBackend(conf map[string]string, logger log.Logger) (Backend, error) { +func buildCouchDBBackend(conf map[string]string, logger log.Logger) (*CouchDBBackend, error) { endpoint := os.Getenv("COUCHDB_ENDPOINT") if endpoint == "" { endpoint = conf["endpoint"] @@ -165,10 +166,15 @@ func newCouchDBBackend(conf map[string]string, logger log.Logger) (Backend, erro password: password, Client: &http.Client{}, }, - logger: logger, + logger: logger, + permitPool: NewPermitPool(DefaultParallelOperations), }, nil } +func newCouchDBBackend(conf map[string]string, logger log.Logger) (Backend, error) { + return buildCouchDBBackend(conf, logger) +} + type couchDBEntry struct { Entry *Entry `json:"entry"` Rev string `json:"_rev,omitempty"` @@ -178,6 +184,9 @@ type couchDBEntry struct { // Put is used to insert or update an entry func (m *CouchDBBackend) Put(entry *Entry) error { + m.permitPool.Acquire() + defer m.permitPool.Release() + defer metrics.MeasureSince([]string{"couchdb", "put"}, time.Now()) revision, _ := m.client.rev(url.PathEscape(entry.Key)) @@ -191,6 +200,9 @@ func (m *CouchDBBackend) Put(entry *Entry) error { // Get is used to fetch an entry func (m *CouchDBBackend) Get(key string) (*Entry, error) { + m.permitPool.Acquire() + defer m.permitPool.Release() + defer metrics.MeasureSince([]string{"couchdb", "get"}, time.Now()) return m.client.get(key) @@ -198,6 +210,9 @@ func (m *CouchDBBackend) Get(key string) (*Entry, error) { // Delete is used to permanently delete an entry func (m *CouchDBBackend) Delete(key string) error { + m.permitPool.Acquire() + defer m.permitPool.Release() + defer metrics.MeasureSince([]string{"couchdb", "delete"}, time.Now()) revision, _ := m.client.rev(url.PathEscape(key)) @@ -235,3 +250,36 @@ func (m *CouchDBBackend) List(prefix string) ([]string, error) { } return out, nil } + +// TransactionalCouchDBBackend creates a couchdb backend that forces all operations to happen +// in serial +type TransactionalCouchDBBackend struct { + CouchDBBackend +} + +func newTransactionalCouchDBBackend(conf map[string]string, logger log.Logger) (Backend, error) { + backend, err := buildCouchDBBackend(conf, logger) + if err != nil { + return nil, err + } + backend.permitPool = NewPermitPool(1) + + return &TransactionalCouchDBBackend{ + CouchDBBackend: *backend, + }, nil +} + +// GetInternal is used to fetch an entry +func (m *TransactionalCouchDBBackend) GetInternal(key string) (*Entry, error) { + return m.Get(key) +} + +// PutInternal is used to insert or update an entry +func (m *TransactionalCouchDBBackend) PutInternal(entry *Entry) error { + return m.Put(entry) +} + +// DeleteInternal is used to permanently delete an entry +func (m *TransactionalCouchDBBackend) DeleteInternal(key string) error { + return m.Delete(key) +} diff --git a/physical/couchdb_test.go b/physical/couchdb_test.go index 132edcb2fcbb..f524641061d7 100644 --- a/physical/couchdb_test.go +++ b/physical/couchdb_test.go @@ -33,6 +33,25 @@ func TestCouchDBBackend(t *testing.T) { testBackend_ListPrefix(t, b) } +func TestTransactionalCouchDBBackend(t *testing.T) { + cleanup, endpoint, username, password := prepareCouchdbDBTestContainer(t) + defer cleanup() + + logger := logformat.NewVaultLogger(log.LevelTrace) + + b, err := NewBackend("couchdb_transactional", logger, map[string]string{ + "endpoint": endpoint, + "username": username, + "password": password, + }) + if err != nil { + t.Fatalf("err: %s", err) + } + + testBackend(t, b) + testBackend_ListPrefix(t, b) +} + func prepareCouchdbDBTestContainer(t *testing.T) (cleanup func(), retAddress, username, password string) { // If environment variable is set, assume caller wants to target a real // DynamoDB. diff --git a/physical/physical.go b/physical/physical.go index 4f9d42d748af..b7524b86c2d4 100644 --- a/physical/physical.go +++ b/physical/physical.go @@ -140,20 +140,21 @@ var builtinBackends = map[string]Factory{ "inmem_transactional_ha": func(_ map[string]string, logger log.Logger) (Backend, error) { return NewTransactionalInmemHA(logger), nil }, - "file_transactional": newTransactionalFileBackend, - "consul": newConsulBackend, - "zookeeper": newZookeeperBackend, - "file": newFileBackend, - "s3": newS3Backend, - "azure": newAzureBackend, - "dynamodb": newDynamoDBBackend, - "etcd": newEtcdBackend, - "mssql": newMsSQLBackend, - "mysql": newMySQLBackend, - "postgresql": newPostgreSQLBackend, - "couchdb": newCouchDBBackend, - "swift": newSwiftBackend, - "gcs": newGCSBackend, + "file_transactional": newTransactionalFileBackend, + "consul": newConsulBackend, + "zookeeper": newZookeeperBackend, + "file": newFileBackend, + "s3": newS3Backend, + "azure": newAzureBackend, + "dynamodb": newDynamoDBBackend, + "etcd": newEtcdBackend, + "mssql": newMsSQLBackend, + "mysql": newMySQLBackend, + "postgresql": newPostgreSQLBackend, + "couchdb": newCouchDBBackend, + "couchdb_transactional": newTransactionalCouchDBBackend, + "swift": newSwiftBackend, + "gcs": newGCSBackend, } // PermitPool is used to limit maximum outstanding requests diff --git a/website/source/docs/configuration/storage/couchdb.html.md b/website/source/docs/configuration/storage/couchdb.html.md index edd756418d73..c65f3a301f28 100644 --- a/website/source/docs/configuration/storage/couchdb.html.md +++ b/website/source/docs/configuration/storage/couchdb.html.md @@ -12,6 +12,9 @@ description: |- The CouchDB storage backend is used to persist Vault's data in [CouchDB][couchdb] table. +- **No High Availability** – the CouchDB backend does not support high + availability. + - **Community Supported** – the CouchDB storage backend is supported by the community. While it has undergone review by HashiCorp employees, they may not be as knowledgeable about the technology. If you encounter problems with them, From bae30ce15dec823100d7353a25f350e362ad5d4e Mon Sep 17 00:00:00 2001 From: Raphael Randschau Date: Fri, 16 Jun 2017 22:25:33 +0200 Subject: [PATCH 3/4] chore(couchdb): correct {Get,Put,Delete}{Internal,} implementation as of the feedback from @jefferai --- physical/couchdb.go | 54 ++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/physical/couchdb.go b/physical/couchdb.go index ff08bec33b2d..3e821053e52f 100644 --- a/physical/couchdb.go +++ b/physical/couchdb.go @@ -187,15 +187,7 @@ func (m *CouchDBBackend) Put(entry *Entry) error { m.permitPool.Acquire() defer m.permitPool.Release() - defer metrics.MeasureSince([]string{"couchdb", "put"}, time.Now()) - - revision, _ := m.client.rev(url.PathEscape(entry.Key)) - - return m.client.put(couchDBEntry{ - Entry: entry, - Rev: revision, - ID: url.PathEscape(entry.Key), - }) + return m.PutInternal(entry) } // Get is used to fetch an entry @@ -203,9 +195,7 @@ func (m *CouchDBBackend) Get(key string) (*Entry, error) { m.permitPool.Acquire() defer m.permitPool.Release() - defer metrics.MeasureSince([]string{"couchdb", "get"}, time.Now()) - - return m.client.get(key) + return m.GetInternal(key) } // Delete is used to permanently delete an entry @@ -213,15 +203,7 @@ func (m *CouchDBBackend) Delete(key string) error { m.permitPool.Acquire() defer m.permitPool.Release() - defer metrics.MeasureSince([]string{"couchdb", "delete"}, time.Now()) - - revision, _ := m.client.rev(url.PathEscape(key)) - deleted := true - return m.client.put(couchDBEntry{ - ID: url.PathEscape(key), - Rev: revision, - Deleted: &deleted, - }) + return m.DeleteInternal(key) } // List is used to list all the keys under a given prefix @@ -270,16 +252,34 @@ func newTransactionalCouchDBBackend(conf map[string]string, logger log.Logger) ( } // GetInternal is used to fetch an entry -func (m *TransactionalCouchDBBackend) GetInternal(key string) (*Entry, error) { - return m.Get(key) +func (m *CouchDBBackend) GetInternal(key string) (*Entry, error) { + defer metrics.MeasureSince([]string{"couchdb", "get"}, time.Now()) + + return m.client.get(key) } // PutInternal is used to insert or update an entry -func (m *TransactionalCouchDBBackend) PutInternal(entry *Entry) error { - return m.Put(entry) +func (m *CouchDBBackend) PutInternal(entry *Entry) error { + defer metrics.MeasureSince([]string{"couchdb", "put"}, time.Now()) + + revision, _ := m.client.rev(url.PathEscape(entry.Key)) + + return m.client.put(couchDBEntry{ + Entry: entry, + Rev: revision, + ID: url.PathEscape(entry.Key), + }) } // DeleteInternal is used to permanently delete an entry -func (m *TransactionalCouchDBBackend) DeleteInternal(key string) error { - return m.Delete(key) +func (m *CouchDBBackend) DeleteInternal(key string) error { + defer metrics.MeasureSince([]string{"couchdb", "delete"}, time.Now()) + + revision, _ := m.client.rev(url.PathEscape(key)) + deleted := true + return m.client.put(couchDBEntry{ + ID: url.PathEscape(key), + Rev: revision, + Deleted: &deleted, + }) } From afc9363ced0a1e925fa9a3f89c691db22ec7b3ac Mon Sep 17 00:00:00 2001 From: Raphael Randschau Date: Sat, 17 Jun 2017 07:39:11 +0200 Subject: [PATCH 4/4] chore(couchdb): use cleanhttp --- physical/couchdb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/physical/couchdb.go b/physical/couchdb.go index 3e821053e52f..c31dc3e8af36 100644 --- a/physical/couchdb.go +++ b/physical/couchdb.go @@ -12,6 +12,7 @@ import ( "time" "github.com/armon/go-metrics" + cleanhttp "github.com/hashicorp/go-cleanhttp" log "github.com/mgutz/logxi/v1" ) @@ -164,7 +165,7 @@ func buildCouchDBBackend(conf map[string]string, logger log.Logger) (*CouchDBBac endpoint: endpoint, username: username, password: password, - Client: &http.Client{}, + Client: cleanhttp.DefaultPooledClient(), }, logger: logger, permitPool: NewPermitPool(DefaultParallelOperations),