files: support encrypt, sign, and encrypt + sign options

.travis.yml

@@ -23,8 +23,8 @@ env:
     - AWS_SECRET_ACCESS_KEY=minioadmin
     - GCS_FAKE_SERVER="https://localhost:4443"
   matrix:
-    - spl=true rel=0.7.13
-    # - spl=false rel=0.8.3
+    - rel=0.7.13
+    # - rel=0.8.3 # some feature preventing zpool creation (encryption)?
 
 before_install:
   - export MAKEFLAGS=-j$(($(grep -c '^processor' /proc/cpuinfo) * 2 + 1))
@@ -34,13 +34,16 @@ before_install:
   - sudo apt-get install -y linux-headers-`uname -r` tree uuid-dev libattr1-dev libblkid-dev jq gnupg2 xz-utils gzip
   - mkdir -p $HOME/zfs
   - cd $HOME/zfs
-  - "[[ -d spl-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/spl-$rel.tar.gz | tar xz"
+  - "[[ -d spl-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/spl-$rel.tar.gz | tar xz || true"
   - "[[ -d zfs-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/zfs-$rel.tar.gz | tar xz"
-  - (cd spl-$rel && ./configure --prefix=/usr && make && sudo make install)
+  - (cd spl-$rel && ./configure --prefix=/usr && make && sudo make install) || true
   - (cd zfs-$rel && ./configure --prefix=/usr && make && sudo make install)
   - sudo modprobe zfs
   - cd $TRAVIS_BUILD_DIR
-  - source ./travis-setup.sh
+  - mkdir temp
+  - export TMPDIR=$PWD/temp
+  - export VDEV=$(mktemp)
+  - chmod +x ./travis-setup.sh && ./travis-setup.sh
 
 install:
   - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $GOPATH/bin v1.24.0
@@ -51,11 +54,11 @@ install:
 script:
   - make build
   - chmod +x ./integration_test.sh && ./integration_test.sh
-  - sudo -E $(which go) test -race -v -coverprofile=coverage.out -covermode=atomic -coverpkg=$(go list ./... | grep -v '/vendor/' | paste -sd, -) ./...  
+  - sudo -E TMPDIR=$TMPDIR $(which go) test -race -v -coverprofile=coverage.out -covermode=atomic -coverpkg=$(go list ./... | grep -v '/vendor/' | paste -sd, -) ./...  
   - make lint
 
 after_success:
   - sudo -E $GOPATH/bin/goveralls -coverprofile=coverage.out -service=travis-ci
 
 after_script: 
-  - source ./travis-teardown.sh
+  - chmod +x ./travis-teardown.sh && ./travis-teardown.sh

backup/backup.go

@@ -21,6 +21,7 @@
 package backup
 
 import (
+	"bytes"
 	"context"
 	"crypto/md5" //nolint:gosec // Not used for cryptography
 	"encoding/json"
@@ -477,10 +478,11 @@ func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInf
 	var group *errgroup.Group
 	group, ctx = errgroup.WithContext(ctx)
 
+	buf := bytes.NewBuffer(nil)
 	cmd := zfs.GetZFSSendCommand(ctx, j)
 	cin, cout := io.Pipe()
 	cmd.Stdout = cout
-	cmd.Stderr = os.Stderr
+	cmd.Stderr = buf
 	counter := datacounter.NewReaderCounter(cin)
 	usingPipe := false
 	if j.MaxFileBuffer == 0 {
@@ -591,7 +593,7 @@ func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInf
 
 	err = group.Wait()
 	if err != nil {
-		log.AppLogger.Errorf("Error waiting for zfs command to finish - %v", err)
+		log.AppLogger.Errorf("Error waiting for zfs command to finish - %v: %s", err, buf.String())
 		return err
 	}
 	log.AppLogger.Infof("zfs send completed without error")

backup/restore.go

@@ -21,6 +21,7 @@
 package backup
 
 import (
+	"bytes"
 	"context"
 	"crypto/md5" // nolint:gosec // MD5 not used for cryptographic purposes here
 	"errors"
@@ -244,32 +245,21 @@ func Receive(pctx context.Context, jobInfo *files.JobInfo) error {
 		}
 	}
 
-	// Compute the Manifest File
-	tempManifest, err := files.CreateManifestVolume(ctx, jobInfo)
-	if err != nil {
-		log.AppLogger.Errorf("Error trying to create manifest volume - %v", err)
-		return err
-	}
-	if err = tempManifest.Close(); err != nil {
-		log.AppLogger.Warningf("Could not close temporary manifest %v", err)
-	}
-	if err = tempManifest.DeleteVolume(); err != nil {
-		log.AppLogger.Warningf("Could not delete temporary manifest %v", err)
-	}
+	manifestObjectName := jobInfo.ManifestObjectName()
 	// nolint:gosec // MD5 not used for cryptographic purposes here
-	safeManifestFile := fmt.Sprintf("%x", md5.Sum([]byte(tempManifest.ObjectName)))
+	safeManifestFile := fmt.Sprintf("%x", md5.Sum([]byte(manifestObjectName)))
 	safeManifestPath := filepath.Join(localCachePath, safeManifestFile)
 
 	// Check to see if we have the manifest file locally
 	manifest, err := readManifest(ctx, safeManifestPath, jobInfo)
 	if err != nil {
 		if os.IsNotExist(err) {
-			if bErr := backend.PreDownload(ctx, []string{tempManifest.ObjectName}); bErr != nil {
-				log.AppLogger.Errorf("Error trying to pre download manifest volume %s - %v", tempManifest.ObjectName, bErr)
+			if bErr := backend.PreDownload(ctx, []string{manifestObjectName}); bErr != nil {
+				log.AppLogger.Errorf("Error trying to pre download manifest volume %s - %v", manifestObjectName, bErr)
 				return bErr
 			}
 			// Try and download the manifest file from the backend
-			if dErr := downloadTo(ctx, backend, tempManifest.ObjectName, safeManifestPath); dErr != nil {
+			if dErr := downloadTo(ctx, backend, manifestObjectName, safeManifestPath); dErr != nil {
 				return dErr
 			}
 			manifest, err = readManifest(ctx, safeManifestPath, jobInfo)
@@ -460,9 +450,10 @@ func processSequence(ctx context.Context, sequence downloadSequence, backend bac
 }
 
 func receiveStream(ctx context.Context, cmd *exec.Cmd, j *files.JobInfo, c <-chan *files.VolumeInfo, buffer <-chan interface{}) error {
+	buf := bytes.NewBuffer(nil)
 	cin, cout := io.Pipe()
 	cmd.Stdin = cin
-	cmd.Stderr = os.Stderr
+	cmd.Stderr = buf
 	var group *errgroup.Group
 	var once sync.Once
 	group, ctx = errgroup.WithContext(ctx)
@@ -533,7 +524,7 @@ func receiveStream(ctx context.Context, cmd *exec.Cmd, j *files.JobInfo, c <-cha
 	// Wait for the command to finish
 	err = group.Wait()
 	if err != nil {
-		log.AppLogger.Errorf("Error waiting for zfs command to finish - %v", err)
+		log.AppLogger.Errorf("Error waiting for zfs command to finish - %v: %s", err, buf.String())
 		return err
 	}
 	log.AppLogger.Infof("zfs receive completed without error")

backup/sync.go

@@ -139,6 +139,7 @@ func syncCache(ctx context.Context, j *files.JobInfo, localCache string, backend
 func validateSnapShotExists(ctx context.Context, snapshot *files.SnapshotInfo, target string, includeBookmarks bool) (bool, error) {
 	snapshots, err := zfs.GetSnapshotsAndBookmarks(ctx, target)
 	if err != nil {
+		log.AppLogger.Debugf("Could not list snapshots for %s: %v", target, err)
 		// TODO: There are some error cases that are ok to ignore!
 		return false, nil
 	}

cmd/clean.go

@@ -21,8 +21,6 @@
 package cmd
 
 import (
-	"context"
-
 	"github.com/spf13/cobra"
 
 	"github.com/someone1/zfsbackup-go/backup"
@@ -39,7 +37,7 @@ var cleanCmd = &cobra.Command{
 	PreRunE:       validateCleanFlags,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		jobInfo.Destinations = []string{args[0]}
-		return backup.Clean(context.Background(), &jobInfo, cleanLocal)
+		return backup.Clean(cmd.Context(), &jobInfo, cleanLocal)
 	},
 }
 

cmd/list.go

@@ -21,7 +21,6 @@
 package cmd
 
 import (
-	"context"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -62,7 +61,7 @@ var listCmd = &cobra.Command{
 		}
 
 		jobInfo.Destinations = []string{args[0]}
-		return backup.List(context.Background(), &jobInfo, startsWith, before, after)
+		return backup.List(cmd.Context(), &jobInfo, startsWith, before, after)
 	},
 }
 

cmd/receive.go

@@ -45,9 +45,9 @@ var receiveCmd = &cobra.Command{
 		log.AppLogger.Infof("Limiting the number of active files to %d", jobInfo.MaxFileBuffer)
 
 		if jobInfo.AutoRestore {
-			return backup.AutoRestore(context.Background(), &jobInfo)
+			return backup.AutoRestore(cmd.Context(), &jobInfo)
 		}
-		return backup.Receive(context.Background(), &jobInfo)
+		return backup.Receive(cmd.Context(), &jobInfo)
 	},
 }
 

cmd/root.go

@@ -21,6 +21,7 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -72,8 +73,8 @@ destination of your choosing.`,
 
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
-func Execute() {
-	if err := RootCmd.Execute(); err != nil {
+func Execute(ctx context.Context) {
+	if err := RootCmd.ExecuteContext(ctx); err != nil {
 		os.Exit(-1)
 	}
 }
@@ -243,9 +244,14 @@ func getAndDecryptPrivateKey(email string) (*openpgp.Entity, error) {
 }
 
 func loadSendKeys() error {
-	if jobInfo.EncryptTo != "" && publicKeyRingPath == "" {
-		log.AppLogger.Errorf("You must specify a public keyring path if you provide an encryptTo option")
-		return errInvalidInput
+	if jobInfo.EncryptTo != "" {
+		if usingSmartOption() && secretKeyRingPath == "" {
+			log.AppLogger.Errorf("You must specify a secret keyring path if you use a smart option with encryptTo")
+			return errInvalidInput
+		} else if publicKeyRingPath == "" {
+			log.AppLogger.Errorf("You must specify a public keyring path if you provide an encryptTo option")
+			return errInvalidInput
+		}
 	}
 
 	if jobInfo.SignFrom != "" && secretKeyRingPath == "" {
@@ -254,7 +260,13 @@ func loadSendKeys() error {
 	}
 
 	if jobInfo.EncryptTo != "" {
-		if jobInfo.EncryptKey = pgp.GetPublicKeyByEmail(jobInfo.EncryptTo); jobInfo.EncryptKey == nil {
+		if usingSmartOption() {
+			var err error
+			jobInfo.EncryptKey, err = getAndDecryptPrivateKey(jobInfo.EncryptTo)
+			if err != nil {
+				return err
+			}
+		} else if jobInfo.EncryptKey = pgp.GetPublicKeyByEmail(jobInfo.EncryptTo); jobInfo.EncryptKey == nil {
 			log.AppLogger.Errorf("Could not find public key for %s", jobInfo.EncryptTo)
 			return errInvalidInput
 		}

cmd/send.go

@@ -63,7 +63,7 @@ var sendCmd = &cobra.Command{
 			log.AppLogger.Infof("Will be signed from %s", jobInfo.SignFrom)
 		}
 
-		return backup.Backup(context.Background(), &jobInfo)
+		return backup.Backup(cmd.Context(), &jobInfo)
 	},
 }
 
@@ -309,12 +309,6 @@ func validateSendFlags(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if usingSmartOption() {
-		if err := loadReceiveKeys(); err != nil {
-			return err
-		}
-	}
-
 	if jobInfo.IncrementalSnapshot.Name != "" && fullIncremental != "" {
 		log.AppLogger.Errorf("The flags -i and -I are mutually exclusive. Please specify only one of these flags.")
 		return errInvalidInput

files/jobinfo.go

@@ -202,3 +202,54 @@ func (j *JobInfo) ValidateSendFlags() error {
 
 	return nil
 }
+
+func (j *JobInfo) ManifestObjectName() string {
+	extensions := []string{"manifest"}
+	nameParts := []string{j.ManifestPrefix}
+
+	baseParts, ext := j.volumeNameParts(true)
+	extensions = append(extensions, ext...)
+	nameParts = append(nameParts, baseParts...)
+
+	return fmt.Sprintf("%s.%s", strings.Join(nameParts, j.Separator), strings.Join(extensions, "."))
+}
+
+func (j *JobInfo) BackupVolumeObjectName(volumeNumber int64) string {
+	extensions := []string{"zstream"}
+
+	nameParts, ext := j.volumeNameParts(false)
+	extensions = append(extensions, ext...)
+	extensions = append(extensions, fmt.Sprintf("vol%d", volumeNumber))
+
+	return fmt.Sprintf("%s.%s", strings.Join(nameParts, j.Separator), strings.Join(extensions, "."))
+}
+
+func (j *JobInfo) volumeNameParts(isManifest bool) (nameParts, extensions []string) {
+	extensions = make([]string, 0, 2)
+
+	if j.EncryptKey != nil || j.SignKey != nil {
+		extensions = append(extensions, "pgp")
+	}
+
+	compressorName := j.Compressor
+	if isManifest {
+		compressorName = InternalCompressor
+	}
+
+	switch compressorName {
+	case InternalCompressor:
+		extensions = append([]string{"gz"}, extensions...)
+	case "", ZfsCompressor:
+	default:
+		extensions = append([]string{compressorName}, extensions...)
+	}
+
+	nameParts = []string{j.VolumeName}
+	if j.IncrementalSnapshot.Name != "" {
+		nameParts = append(nameParts, j.IncrementalSnapshot.Name, "to", j.BaseSnapshot.Name)
+	} else {
+		nameParts = append(nameParts, j.BaseSnapshot.Name)
+	}
+
+	return nameParts, extensions
+}