Add MySQL storage and tests

.github/workflows/go.yml

@@ -24,6 +24,15 @@ jobs:
 
     - name: Build
       run: go build -v ./...
+
+    - name: Start test docker containers
+      if: matrix.platform == 'ubuntu-latest'
+      run: |
+        docker-compose -f storage/mysql/docker-compose-test.yml up &
+
+    - name: Set 
+      if: matrix.platform == 'ubuntu-latest'
+      run: echo "NANODEP_MYSQL_STORAGE_TEST=1" >> $GITHUB_ENV
 
     - name: Test
       run: go test -v -race ./...

Makefile

@@ -62,4 +62,4 @@ release: \
 test:
 	go test -v -cover -race ./...
 
-.PHONY: my docker $(DEPTOKENS) $(DEPSERVER) $(DEPSYNCER) clean release test
+.PHONY: my docker $(DEPTOKENS) $(DEPSERVER) $(DEPSYNCER) clean release test

client/client.go

@@ -1,4 +1,4 @@
-// Package client implements HTTP privitives for talking with and authenticating with the Apple DEP APIs.
+// Package client implements HTTP primitives for talking with and authenticating with the Apple DEP APIs.
 package client
 
 import (

client/transport.go

@@ -41,7 +41,7 @@ func GetName(ctx context.Context) string {
 }
 
 type AuthTokensRetriever interface {
-	RetrieveAuthTokens(context.Context, string) (*OAuth1Tokens, error)
+	RetrieveAuthTokens(ctx context.Context, name string) (*OAuth1Tokens, error)
 }
 
 type SessionStore interface {

cmd/depserver/main.go

@@ -10,10 +10,10 @@ import (
 	"time"
 
 	"github.com/micromdm/nanodep/client"
-	"github.com/micromdm/nanodep/cmd"
 	dephttp "github.com/micromdm/nanodep/http"
 	"github.com/micromdm/nanodep/http/api"
 	"github.com/micromdm/nanodep/log/stdlogfmt"
+	"github.com/micromdm/nanodep/parse"
 	"github.com/micromdm/nanodep/proxy"
 )
 
@@ -37,7 +37,7 @@ func main() {
 		flListen  = flag.String("listen", ":9001", "HTTP listen address")
 		flAPIKey  = flag.String("api", "", "API key for API endpoints")
 		flVersion = flag.Bool("version", false, "print version")
-		flStorage = flag.String("storage", "", "storage backend")
+		flStorage = flag.String("storage", "file", "storage backend")
 		flDSN     = flag.String("storage-dsn", "", "storage data source name")
 	)
 	flag.Parse()
@@ -55,7 +55,7 @@ func main() {
 
 	logger := stdlogfmt.New(stdlog.Default(), *flDebug)
 
-	storage, err := cmd.ParseStorage(*flStorage, *flDSN)
+	storage, err := parse.Storage(*flStorage, *flDSN)
 	if err != nil {
 		logger.Info("msg", "creating storage backend", "err", err)
 		os.Exit(1)

cmd/depsyncer/main.go

@@ -12,9 +12,9 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/micromdm/nanodep/cmd"
 	"github.com/micromdm/nanodep/godep"
 	"github.com/micromdm/nanodep/log/stdlogfmt"
+	"github.com/micromdm/nanodep/parse"
 	depsync "github.com/micromdm/nanodep/sync"
 )
 
@@ -30,7 +30,7 @@ func main() {
 		flLimit   = flag.Int("limit", 0, "limit fetch and sync calls to this many devices (0 for server default)")
 		flDebug   = flag.Bool("debug", false, "log debug messages")
 		flADebug  = flag.Bool("debug-assigner", false, "additional debug logging of the device assigner")
-		flStorage = flag.String("storage", "", "storage backend")
+		flStorage = flag.String("storage", "file", "storage backend")
 		flDSN     = flag.String("storage-dsn", "", "storage data source name")
 		flWebhook = flag.String("webhook-url", "", "URL to send requests to")
 	)
@@ -53,7 +53,7 @@ func main() {
 
 	logger := stdlogfmt.New(stdlog.Default(), *flDebug)
 
-	storage, err := cmd.ParseStorage(*flStorage, *flDSN)
+	storage, err := parse.Storage(*flStorage, *flDSN)
 	if err != nil {
 		logger.Info("msg", "creating storage backend", "err", err)
 		os.Exit(1)

docs/operations-guide.md

@@ -49,6 +49,15 @@ Configure the `file` storage backend. This backend manages DEP authentication an
 
 *Example:* `-storage file -storage-dsn /path/to/my/db`
 
+##### mysql storage backend
+
+* `-storage mysql`
+
+Configures the MySQL storage backend. The `-dsn` flag should be in the [format the SQL driver expects](https://github.com/go-sql-driver/mysql#dsn-data-source-name).
+Be sure to create the storage tables with the [schema.sql](../storage/mysql/schema.sql) file. MySQL 8.0.19 or later is required.
+
+*Example:* `-storage mysql -dsn nanodep:nanodep/mydepdb`
+
 #### -version
 
 * print version

go.mod

@@ -5,3 +5,5 @@ go 1.17
 require go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352
 
 require github.com/gomodule/oauth1 v0.2.0
+
+require github.com/go-sql-driver/mysql v1.6.0

go.sum

@@ -1,3 +1,5 @@
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/gomodule/oauth1 v0.2.0 h1:/nNHAD99yipOEspQFbAnNmwGTZ1UNXiD/+JLxwx79fo=
 github.com/gomodule/oauth1 v0.2.0/go.mod h1:4r/a8/3RkhMBxJQWL5qzbOEcaQmNPIkNoI7P8sXeI08=
 go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak=

http/api/assigner.go

@@ -45,7 +45,7 @@ func RetrieveAssignerProfileHandler(store sync.AssignerProfileRetriever, logger
 }
 
 type AssignerProfileStorer interface {
-	StoreAssignerProfile(context.Context, string, string) error
+	StoreAssignerProfile(ctx context.Context, name string, profileUUID string) error
 }
 
 // StoreAssignerProfileHandler saves the assigner profile UUID for the

http/api/config.go

@@ -41,7 +41,7 @@ func RetrieveConfigHandler(store client.ConfigRetriever, logger log.Logger) http
 }
 
 type ConfigStorer interface {
-	StoreConfig(context.Context, string, *client.Config) error
+	StoreConfig(ctx context.Context, name string, config *client.Config) error
 }
 
 // StoreConfigHandler stores the DEP server config for the DEP

http/api/tokenpki.go

@@ -2,11 +2,7 @@ package api
 
 import (
 	"context"
-	"crypto/rsa"
-	"crypto/x509"
 	"encoding/json"
-	"encoding/pem"
-	"errors"
 	"io"
 	"net/http"
 
@@ -17,27 +13,18 @@ import (
 )
 
 type TokenPKIRetriever interface {
-	RetrieveTokenPKI(context.Context, string) ([]byte, []byte, error)
+	RetrieveTokenPKI(ctx context.Context, name string) (pemCert []byte, pemKey []byte, err error)
 }
 
 type TokenPKIStorer interface {
-	StoreTokenPKI(context.Context, string, []byte, []byte) error
+	StoreTokenPKI(ctx context.Context, name string, pemCert []byte, pemKey []byte) error
 }
 
 const (
 	defaultCN   = "depserver"
 	defaultDays = 1
 )
 
-// PEMRSAPrivateKey returns key as a PEM block.
-func PEMRSAPrivateKey(key *rsa.PrivateKey) []byte {
-	block := &pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: x509.MarshalPKCS1PrivateKey(key),
-	}
-	return pem.EncodeToMemory(block)
-}
-
 // GetCertTokenPKIHandler generates a new private key and certificate for
 // the token PKI exchange with the ABM/ASM/BE portal. Every call to this
 // handler generates a new keypair and stores it. The PEM-encoded certificate
@@ -62,7 +49,7 @@ func GetCertTokenPKIHandler(store TokenPKIStorer, logger log.Logger) http.Handle
 			return
 		}
 		pemCert := tokenpki.PEMCertificate(cert.Raw)
-		err = store.StoreTokenPKI(r.Context(), r.URL.Path, pemCert, PEMRSAPrivateKey(key))
+		err = store.StoreTokenPKI(r.Context(), r.URL.Path, pemCert, tokenpki.PEMRSAPrivateKey(key))
 		if err != nil {
 			logger.Info("msg", "storing token keypair", "err", err)
 			jsonError(w, err)
@@ -74,15 +61,6 @@ func GetCertTokenPKIHandler(store TokenPKIStorer, logger log.Logger) http.Handle
 	}
 }
 
-// RSAKeyFromPEM decodes a PEM RSA private key.
-func RSAKeyFromPEM(key []byte) (*rsa.PrivateKey, error) {
-	block, _ := pem.Decode(key)
-	if block.Type != "RSA PRIVATE KEY" {
-		return nil, errors.New("PEM type is not RSA PRIVATE KEY")
-	}
-	return x509.ParsePKCS1PrivateKey(block.Bytes)
-}
-
 // DecryptTokenPKIHandler reads the Apple-provided encrypted token ".p7m" file
 // from the request body and decrypts it with the keypair generated from
 // GetCertTokenPKIHandler.
@@ -118,7 +96,7 @@ func DecryptTokenPKIHandler(store TokenPKIRetriever, tokenStore AuthTokensStorer
 			jsonError(w, err)
 			return
 		}
-		key, err := RSAKeyFromPEM(keyBytes)
+		key, err := tokenpki.RSAKeyFromPEM(keyBytes)
 		if err != nil {
 			logger.Info("msg", "decoding retrieved private key", "err", err)
 			jsonError(w, err)

http/api/tokens.go

@@ -11,7 +11,7 @@ import (
 )
 
 type AuthTokensStorer interface {
-	StoreAuthTokens(context.Context, string, *client.OAuth1Tokens) error
+	StoreAuthTokens(ctx context.Context, name string, tokens *client.OAuth1Tokens) error
 }
 
 // RetrieveAuthTokensHandler returns the DEP server OAuth1 tokens for the DEP

parse/storage.go

@@ -0,0 +1,28 @@
+package parse
+
+import (
+	"fmt"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/micromdm/nanodep/storage"
+	"github.com/micromdm/nanodep/storage/file"
+	"github.com/micromdm/nanodep/storage/mysql"
+)
+
+// Storage parses a storage name and dsn to determine which and return a storage backend.
+func Storage(storageName, dsn string) (storage.AllStorage, error) {
+	var store storage.AllStorage
+	var err error
+	switch storageName {
+	case "file":
+		if dsn == "" {
+			dsn = "db"
+		}
+		store, err = file.New(dsn)
+	case "mysql":
+		store, err = mysql.New(mysql.WithDSN(dsn))
+	default:
+		return nil, fmt.Errorf("unknown storage: %q", storageName)
+	}
+	return store, err
+}

storage/file/file.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/micromdm/nanodep/client"
+	"github.com/micromdm/nanodep/storage"
 )
 
 const defaultFileMode = 0644
@@ -20,6 +21,8 @@ type FileStorage struct {
 	path string
 }
 
+var _ storage.AllStorage = (*FileStorage)(nil)
+
 // New creates a new FileStorage backend.
 func New(path string) (*FileStorage, error) {
 	err := os.Mkdir(path, 0755)
@@ -62,10 +65,17 @@ func (s *FileStorage) tokenpkiFilename(name, kind string) string {
 // RetrieveAuthTokens reads the JSON DEP OAuth tokens from disk for name DEP name.
 func (s *FileStorage) RetrieveAuthTokens(_ context.Context, name string) (*client.OAuth1Tokens, error) {
 	tokens := new(client.OAuth1Tokens)
-	return tokens, decodeJSONfile(s.tokensFilename(name), tokens)
+	err := decodeJSONfile(s.tokensFilename(name), tokens)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, storage.ErrNotFound
+		}
+		return nil, err
+	}
+	return tokens, nil
 }
 
-// RetrieveAuthTokens saves the DEP OAuth tokens to disk as JSON for name DEP name.
+// StoreAuthTokens saves the DEP OAuth tokens to disk as JSON for name DEP name.
 func (s *FileStorage) StoreAuthTokens(_ context.Context, name string, tokens *client.OAuth1Tokens) error {
 	f, err := os.Create(s.tokensFilename(name))
 	if err != nil {
@@ -85,8 +95,8 @@ func decodeJSONfile(filename string, v interface{}) error {
 }
 
 // RetrieveConfig reads the JSON DEP config from disk for name DEP name.
-// We return an empty config if the config does not exist on disk.
-// This is to support a fallback default config.
+//
+// Returns an empty config if the config does not exist (to support a fallback default config).
 func (s *FileStorage) RetrieveConfig(_ context.Context, name string) (*client.Config, error) {
 	config := new(client.Config)
 	err := decodeJSONfile(s.configFilename(name), config)
@@ -109,6 +119,8 @@ func (s *FileStorage) StoreConfig(_ context.Context, name string, config *client
 
 // RetrieveAssignerProfile reads the assigner profile UUID and its configured
 // timestamp from disk for name DEP name.
+//
+// Returns an empty profile if it does not exist.
 func (s *FileStorage) RetrieveAssignerProfile(_ context.Context, name string) (string, time.Time, error) {
 	profileBytes, err := os.ReadFile(s.profileFilename(name))
 	if err != nil && errors.Is(err, os.ErrNotExist) {
@@ -119,7 +131,7 @@ func (s *FileStorage) RetrieveAssignerProfile(_ context.Context, name string) (s
 	if err == nil {
 		var stat fs.FileInfo
 		stat, err = os.Stat(s.profileFilename(name))
-		if err != nil {
+		if err == nil {
 			modTime = stat.ModTime()
 		}
 	}
@@ -164,10 +176,16 @@ func (s *FileStorage) StoreTokenPKI(_ context.Context, name string, pemCert []by
 func (s *FileStorage) RetrieveTokenPKI(_ context.Context, name string) ([]byte, []byte, error) {
 	certBytes, err := os.ReadFile(s.tokenpkiFilename(name, "cert"))
 	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil, storage.ErrNotFound
+		}
 		return nil, nil, err
 	}
 	keyBytes, err := os.ReadFile(s.tokenpkiFilename(name, "key"))
 	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil, storage.ErrNotFound
+		}
 		return nil, nil, err
 	}
 	return certBytes, keyBytes, err