new inmem and filekv storage backends (#68)

cli/storage.go

@@ -2,24 +2,37 @@
 package cli
 
 import (
+	"errors"
 	"fmt"
 
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/micromdm/nanodep/storage"
+	"github.com/micromdm/nanodep/storage/diskv"
 	"github.com/micromdm/nanodep/storage/file"
+	"github.com/micromdm/nanodep/storage/inmem"
 	"github.com/micromdm/nanodep/storage/mysql"
 )
 
 // Storage parses a storage name and dsn to determine which and return a storage backend.
-func Storage(storageName, dsn string) (storage.AllStorage, error) {
+func Storage(storageName, dsn, options string) (storage.AllStorage, error) {
 	var store storage.AllStorage
 	var err error
 	switch storageName {
+	case "filekv":
+		if dsn == "" {
+			dsn = "dbkv"
+		}
+		store = diskv.New(dsn)
 	case "file":
+		if options != "enable_deprecated=1" {
+			return nil, errors.New("file backend is deprecated; specify storage options to force enable")
+		}
 		if dsn == "" {
 			dsn = "db"
 		}
 		store, err = file.New(dsn)
+	case "inmem":
+		store = inmem.New()
 	case "mysql":
 		store, err = mysql.New(mysql.WithDSN(dsn))
 	default:

client/client.go

@@ -21,10 +21,9 @@ type Config struct {
 }
 
 type ConfigRetriever interface {
-	// RetrieveConfig reads the JSON DEP config of a DEP name.
-	//
-	// Returns (nil, nil) if the DEP name does not exist, or if the config
-	// for the DEP name does not exist.
+	// RetrieveConfig retrieves config of name (DEP name).
+	// If the DEP name or config does not exist then a nil config and
+	// nil error should be returned.
 	RetrieveConfig(ctx context.Context, name string) (*Config, error)
 }
 

client/transport.go

@@ -41,6 +41,8 @@ func GetName(ctx context.Context) string {
 }
 
 type AuthTokensRetriever interface {
+	// RetrieveAuthTokens retrieves the OAuth tokens from storage for name (DEP name).
+	// If the name or tokens do not exist storage.ErrNotFound should be returned.
 	RetrieveAuthTokens(ctx context.Context, name string) (*OAuth1Tokens, error)
 }
 

cmd/depserver/main.go

@@ -38,8 +38,9 @@ func main() {
 		flListen  = flag.String("listen", ":9001", "HTTP listen address")
 		flAPIKey  = flag.String("api", "", "API key for API endpoints")
 		flVersion = flag.Bool("version", false, "print version")
-		flStorage = flag.String("storage", "file", "storage backend")
-		flDSN     = flag.String("storage-dsn", "", "storage data source name")
+		flStorage = flag.String("storage", "filekv", "storage backend")
+		flDSN     = flag.String("storage-dsn", "", "storage backend data source name")
+		flOptions = flag.String("storage-options", "", "storage backend options")
 	)
 	flag.Parse()
 
@@ -59,7 +60,7 @@ func main() {
 		stdlogfmt.WithDebugFlag(*flDebug),
 	)
 
-	storage, err := cli.Storage(*flStorage, *flDSN)
+	storage, err := cli.Storage(*flStorage, *flDSN, *flOptions)
 	if err != nil {
 		logger.Info("msg", "creating storage backend", "err", err)
 		os.Exit(1)

cmd/depsyncer/main.go

@@ -31,8 +31,9 @@ func main() {
 		flDebug   = flag.Bool("debug", false, "log debug messages")
 		flADebug  = flag.Bool("debug-assigner", false, "additional debug logging of the device assigner")
 		flSDebug  = flag.Bool("debug-syncer", false, "additional debug logging of the device syncer")
-		flStorage = flag.String("storage", "file", "storage backend")
-		flDSN     = flag.String("storage-dsn", "", "storage data source name")
+		flStorage = flag.String("storage", "filekv", "storage backend")
+		flDSN     = flag.String("storage-dsn", "", "storage backend data source name")
+		flOptions = flag.String("storage-options", "", "storage backend options")
 		flWebhook = flag.String("webhook-url", "", "URL to send requests to")
 		flUA      = flag.String("user-agent", godep.UserAgent, "User-Agent string to use")
 	)
@@ -58,7 +59,7 @@ func main() {
 		stdlogfmt.WithDebugFlag(*flDebug),
 	)
 
-	storage, err := cli.Storage(*flStorage, *flDSN)
+	storage, err := cli.Storage(*flStorage, *flDSN, *flOptions)
 	if err != nil {
 		logger.Info("msg", "creating storage backend", "err", err)
 		os.Exit(1)

docs/operations-guide.md

@@ -37,18 +37,44 @@ Enable additional debug logging.
 
 Specifies the listen address (interface and port number) for the server to listen on.
 
-#### -storage & -storage-dsn
+#### -storage, -storage-dsn, & -storage-options
+
+The `-storage`, `-storage-dsn`, and `-storage-options` flags together configure the storage backend. `-storage` specifies the name of backend type while `-storage-dsn` specifies the backend data source name (e.g. the connection string). The optional `-storage-options` flag specifies options for the backend if it supports them. If no `-storage` backend is specified then `filekv` is used as a default. NanoDEP versions before v0.4 defaulted to the `file` backend.
+
+##### filekv storage backend
+
+* `-storage filekv`
 
-The `-storage` and `-storage-dsn` flags together configure the storage backend. `-storage` specifies the name of backend type while `-storage-dsn` specifies the backend data source name (e.g. the connection string). If no `-storage` backend is specified then `file` is used as a default.
+Configure the `filekv` storage backend. This backend manages DEP authentication and configuration data within plain filesystem files and directories using a key-value storage system. It has zero dependencies, no options, and should run out of the box. The `-storage-dsn` flag specifies the filesystem directory for the database. If no `storage-dsn` is specified then `dbkv` is used as a default.
+
+*Example:* `-storage filekv -storage-dsn /path/to/my/db`
 
 ##### file storage backend
 
 * `-storage file`
 
-Configure the `file` storage backend. This backend manages DEP authentication and configuration data within plain filesystem files and directories. It has zero dependencies and should run out of the box. The `-storage-dsn` flag specifies the filesystem directory for the database. If no `storage-dsn` is specified then `db` is used as a default.
+> [!WARNING]
+> The `file` storage backend is deprecated in NanoDEP v0.4 and will be removed in a future release.
+
+Configure the `file` storage backend. This backend manages DEP authentication and configuration data within plain filesystem files and directories. It has zero dependencies and and should run out of the box. The `-storage-dsn` flag specifies the filesystem directory for the database. If no `storage-dsn` is specified then `db` is used as a default.
 
 *Example:* `-storage file -storage-dsn /path/to/my/db`
 
+Options are specified as a comma-separated list of "key=value" pairs. Supported options:
+
+* `enable_deprecated=1`
+  * This option enables the file backend. Without this switch the `file` backend is disabled.
+
+*Example:* `-storage file -storage-dsn /path/to/my/db -storage-options enable_deprecated=1`
+
+##### in-memory storage backend
+
+* `-storage inmem`
+
+Configure the `inmem` in-memory storage backend. This backend manages DEP authentication and configuration data entirely in volatile memory. There are no options and the DSN is ignored. **WARNING: all data is lost when the server or tool process has exited.**
+
+*Example:* `-storage inmem`
+
 ##### mysql storage backend
 
 * `-storage mysql`

go.mod

@@ -5,8 +5,12 @@ go 1.17
 require (
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/gomodule/oauth1 v0.2.0
-	github.com/micromdm/nanolib v0.1.1
+	github.com/micromdm/nanolib v0.2.0
+	github.com/peterbourgon/diskv/v3 v3.0.1
 	github.com/smallstep/pkcs7 v0.0.0-20231107075624-be1870d87d13
 )
 
-require filippo.io/edwards25519 v1.1.0 // indirect
+require (
+	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/google/btree v1.0.0 // indirect
+)

go.sum

@@ -4,9 +4,11 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/gomodule/oauth1 v0.2.0 h1:/nNHAD99yipOEspQFbAnNmwGTZ1UNXiD/+JLxwx79fo=
 github.com/gomodule/oauth1 v0.2.0/go.mod h1:4r/a8/3RkhMBxJQWL5qzbOEcaQmNPIkNoI7P8sXeI08=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/micromdm/nanolib v0.1.1 h1:nNwY2xLBTHSpwEJsW5xGjkW9MdskAbeo/e6+ZYwr2mE=
-github.com/micromdm/nanolib v0.1.1/go.mod h1:FwBKCvvphgYvbdUZ+qw5kay7NHJcg6zPi8W7kXNajmE=
+github.com/micromdm/nanolib v0.2.0 h1:g5GHQuUpS82WIAB15LyenjF/0/WSUNJMe5XZfCJSXq4=
+github.com/micromdm/nanolib v0.2.0/go.mod h1:FwBKCvvphgYvbdUZ+qw5kay7NHJcg6zPi8W7kXNajmE=
+github.com/peterbourgon/diskv/v3 v3.0.1 h1:x06SQA46+PKIUftmEujdwSEpIx8kR+M9eLYsUxeYveU=
 github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o=
 github.com/smallstep/pkcs7 v0.0.0-20231107075624-be1870d87d13 h1:qRxEt9ESQhAg1kjmgJ8oyyzlc9zkAjOooe7bcKjKORQ=
 github.com/smallstep/pkcs7 v0.0.0-20231107075624-be1870d87d13/go.mod h1:SoUAr/4M46rZ3WaLstHxGhLEgoYIDRqxQEXLOmOEB0Y=

http/api/assigner.go

@@ -46,6 +46,7 @@ func RetrieveAssignerProfileHandler(store sync.AssignerProfileRetriever, logger
 }
 
 type AssignerProfileStorer interface {
+	// StoreAssignerProfile stores the assigner profile UUID for name (DEP name).
 	StoreAssignerProfile(ctx context.Context, name string, profileUUID string) error
 }
 

http/api/config.go

@@ -43,6 +43,8 @@ func RetrieveConfigHandler(store client.ConfigRetriever, logger log.Logger) http
 }
 
 type ConfigStorer interface {
+	// StoreConfig stores config for name (DEP name).
+	// The entire config is overwritten.
 	StoreConfig(ctx context.Context, name string, config *client.Config) error
 }
 

http/api/tokenpki.go

@@ -18,18 +18,28 @@ import (
 )
 
 type TokenPKIStagingRetriever interface {
+	// RetrieveStagingTokenPKI retrieves and returns the PEM bytes for the staged
+	// DEP token exchange certificate and private key for name (DEP name).
 	RetrieveStagingTokenPKI(ctx context.Context, name string) (pemCert []byte, pemKey []byte, err error)
 }
 
 type TokenPKICurrentRetriever interface {
+	// RetrieveCurrentTokenPKI reads and returns the PEM bytes for the
+	// previously-upstaged DEP token exchange certificate and private
+	// key using name (DEP name).
 	RetrieveCurrentTokenPKI(ctx context.Context, name string) (pemCert []byte, pemKey []byte, err error)
 }
 
 type TokenPKIUpstager interface {
+	// UpstageTokenPKI copies the "staging" PKI certificate and key to the current PKI certificate and key.
+	// This allows key operations to use the newly uploaded key.
+	// Note the OAuth tokens should also be changed at the same time.
 	UpstageTokenPKI(ctx context.Context, name string) error
 }
 
 type TokenPKIStorer interface {
+	// StoreTokenPKI stores the PEM bytes in pemCert and pemKey for name (DEP name).
+	// These will be stored as the "staging" set to later be "upstaged."
 	StoreTokenPKI(ctx context.Context, name string, pemCert []byte, pemKey []byte) error
 }
 

http/api/tokens.go

@@ -25,6 +25,8 @@ type AuthTokensStore interface {
 }
 
 type AuthTokensStorer interface {
+	// StoreAuthTokens saves the DEP OAuth tokens for name (DEP name).
+	// Note the staging DEP certificate and key should also be upstaged at the same time.
 	StoreAuthTokens(ctx context.Context, name string, tokens *client.OAuth1Tokens) error
 }
 

storage/diskv/diskv.go

@@ -0,0 +1,28 @@
+// Package diskv implements a NanoDEP storage backend using the diskv key-value store.
+package diskv
+
+import (
+	"path/filepath"
+
+	"github.com/micromdm/nanodep/storage/kv"
+
+	"github.com/micromdm/nanolib/storage/kv/kvdiskv"
+	"github.com/micromdm/nanolib/storage/kv/kvtxn"
+	"github.com/peterbourgon/diskv/v3"
+)
+
+// Diskv is a storage backend that uses diskv.
+type Diskv struct {
+	*kv.KV
+}
+
+// New creates a new storage backend that uses diskv.
+func New(path string) *Diskv {
+	return &Diskv{KV: kv.New(
+		kvtxn.New(kvdiskv.New(diskv.New(diskv.Options{
+			BasePath:     filepath.Join(path, "dep_names"),
+			Transform:    kvdiskv.FlatTransform,
+			CacheSizeMax: 1024 * 1024,
+		}))),
+	)}
+}

storage/diskv/diskv_test.go

@@ -0,0 +1,12 @@
+package diskv
+
+import (
+	"context"
+	"testing"
+
+	"github.com/micromdm/nanodep/storage/test"
+)
+
+func TestFileStorage(t *testing.T) {
+	test.TestWithStorages(t, context.Background(), New(t.TempDir()))
+}

storage/file/file.go

@@ -62,7 +62,7 @@ func (s *FileStorage) tokenpkiFilename(name, kind string) string {
 	return path.Join(s.path, name+".tokenpki."+kind+".txt")
 }
 
-// RetrieveAuthTokens reads the JSON DEP OAuth tokens from disk for name DEP name.
+// RetrieveAuthTokens reads the JSON DEP OAuth tokens from disk for name (DEP name).
 func (s *FileStorage) RetrieveAuthTokens(_ context.Context, name string) (*client.OAuth1Tokens, error) {
 	tokens := new(client.OAuth1Tokens)
 	err := decodeJSONfile(s.tokensFilename(name), tokens)
@@ -72,7 +72,7 @@ func (s *FileStorage) RetrieveAuthTokens(_ context.Context, name string) (*clien
 	return tokens, err
 }
 
-// StoreAuthTokens saves the DEP OAuth tokens to disk as JSON for name DEP name.
+// StoreAuthTokens saves the DEP OAuth tokens to disk as JSON for name (DEP name).
 func (s *FileStorage) StoreAuthTokens(_ context.Context, name string, tokens *client.OAuth1Tokens) error {
 	f, err := os.Create(s.tokensFilename(name))
 	if err != nil {
@@ -105,7 +105,7 @@ func (s *FileStorage) RetrieveConfig(_ context.Context, name string) (*client.Co
 	return config, err
 }
 
-// StoreConfig saves the DEP config to disk as JSON for name DEP name.
+// StoreConfig saves the DEP config to disk as JSON for name (DEP name).
 func (s *FileStorage) StoreConfig(_ context.Context, name string, config *client.Config) error {
 	f, err := os.Create(s.configFilename(name))
 	if err != nil {
@@ -116,7 +116,7 @@ func (s *FileStorage) StoreConfig(_ context.Context, name string, config *client
 }
 
 // RetrieveAssignerProfile reads the assigner profile UUID and its configured
-// timestamp from disk for name DEP name.
+// timestamp from disk for name (DEP name).
 //
 // Returns an empty profile if it does not exist.
 func (s *FileStorage) RetrieveAssignerProfile(_ context.Context, name string) (string, time.Time, error) {
@@ -136,13 +136,13 @@ func (s *FileStorage) RetrieveAssignerProfile(_ context.Context, name string) (s
 	return strings.TrimSpace(string(profileBytes)), modTime, err
 }
 
-// StoreAssignerProfile saves the assigner profile UUID to disk for name DEP name.
+// StoreAssignerProfile saves the assigner profile UUID to disk for name (DEP name).
 func (s *FileStorage) StoreAssignerProfile(_ context.Context, name string, profileUUID string) error {
 	return os.WriteFile(s.profileFilename(name), []byte(profileUUID+"\n"), defaultFileMode)
 }
 
 // RetrieveCursor reads the reads the DEP fetch and sync cursor from disk
-// for name DEP name. We return an empty cursor if the cursor does not exist
+// for name (DEP name). We return an empty cursor if the cursor does not exist
 // on disk.
 func (s *FileStorage) RetrieveCursor(_ context.Context, name string) (string, error) {
 	cursorBytes, err := os.ReadFile(s.cursorFilename(name))
@@ -153,12 +153,12 @@ func (s *FileStorage) RetrieveCursor(_ context.Context, name string) (string, er
 	return strings.TrimSpace(string(cursorBytes)), err
 }
 
-// StoreCursor saves the DEP fetch and sync cursor to disk for name DEP name.
+// StoreCursor saves the DEP fetch and sync cursor to disk for name (DEP name).
 func (s *FileStorage) StoreCursor(_ context.Context, name, cursor string) error {
 	return os.WriteFile(s.cursorFilename(name), []byte(cursor+"\n"), defaultFileMode)
 }
 
-// StoreTokenPKI stores the PEM bytes in pemCert and pemKey to disk for name DEP name.
+// StoreTokenPKI stores the PEM bytes in pemCert and pemKey to disk for name (DEP name).
 func (s *FileStorage) StoreTokenPKI(_ context.Context, name string, pemCert []byte, pemKey []byte) error {
 	if err := os.WriteFile(s.tokenpkiFilename(name, "staging.cert"), pemCert, 0664); err != nil {
 		return err
@@ -204,20 +204,20 @@ func (s *FileStorage) UpstageTokenPKI(ctx context.Context, name string) error {
 }
 
 // RetrieveStagingTokenPKI reads and returns the PEM bytes for the staged
-// DEP token exchange certificate and private key from disk using name DEP name.
+// DEP token exchange certificate and private key from disk using name (DEP name).
 func (s *FileStorage) RetrieveStagingTokenPKI(ctx context.Context, name string) ([]byte, []byte, error) {
 	return s.retrieveTokenPKIExtn(name, "staging.")
 }
 
 // RetrieveCurrentTokenPKI reads and returns the PEM bytes for the previously-
 // upstaged DEP token exchange certificate and private key from disk using
-// name DEP name.
+// name (DEP name).
 func (s *FileStorage) RetrieveCurrentTokenPKI(_ context.Context, name string) ([]byte, []byte, error) {
 	return s.retrieveTokenPKIExtn(name, "")
 }
 
 // retrieveTokenPKIExtn reads and returns the PEM bytes for the DEP token exchange
-// certificate and private key from disk using name DEP name and extn type.
+// certificate and private key from disk using name (DEP name) and extn type.
 func (s *FileStorage) retrieveTokenPKIExtn(name, extn string) ([]byte, []byte, error) {
 	certBytes, err := os.ReadFile(s.tokenpkiFilename(name, extn+"cert"))
 	if err != nil {

storage/inmem/inmem.go

@@ -0,0 +1,19 @@
+// Package inmem implements an in-memory NanoDEP storage backend.
+package inmem
+
+import (
+	"github.com/micromdm/nanodep/storage/kv"
+
+	"github.com/micromdm/nanolib/storage/kv/kvmap"
+	"github.com/micromdm/nanolib/storage/kv/kvtxn"
+)
+
+// InMem is an in-memory storage backend.
+type InMem struct {
+	*kv.KV
+}
+
+// New creates a new in-memory storage backend.
+func New() *InMem {
+	return &InMem{KV: kv.New(kvtxn.New(kvmap.New()))}
+}