From 50608b64b1bf2ce57e3305d4b6d956377fc92a01 Mon Sep 17 00:00:00 2001
From: Prateek Malhotra <someone1@gmail.com>
Date: Sun, 3 May 2020 02:22:35 -0400
Subject: [PATCH 1/7] files: support encrypt, sign, and encrypt + sign options

---
 .travis.yml         |   8 +--
 backup/backup.go    |   6 +-
 backup/restore.go   |  27 +++------
 cmd/clean.go        |   4 +-
 cmd/list.go         |   3 +-
 cmd/receive.go      |   4 +-
 cmd/root.go         |  24 ++++++--
 cmd/send.go         |   8 +--
 files/jobinfo.go    |  51 +++++++++++++++++
 files/volumeinfo.go |  61 ++++++++------------
 integration_test.go | 134 +++++++++++++++++++++++++++++++++++---------
 main.go             |   7 ++-
 travis-setup.sh     |   3 +-
 travis-teardown.sh  |   2 +-
 14 files changed, 231 insertions(+), 111 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 221ad3b..8502d5f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,8 +23,8 @@ env:
     - AWS_SECRET_ACCESS_KEY=minioadmin
     - GCS_FAKE_SERVER="https://localhost:4443"
   matrix:
-    - spl=true rel=0.7.13
-    # - spl=false rel=0.8.3
+    - rel=0.7.13
+    - rel=0.8.3
 
 before_install:
   - export MAKEFLAGS=-j$(($(grep -c '^processor' /proc/cpuinfo) * 2 + 1))
@@ -35,8 +35,8 @@ before_install:
   - mkdir -p $HOME/zfs
   - cd $HOME/zfs
   - "[[ -d spl-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/spl-$rel.tar.gz | tar xz"
-  - "[[ -d zfs-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/zfs-$rel.tar.gz | tar xz"
-  - (cd spl-$rel && ./configure --prefix=/usr && make && sudo make install)
+  - "[[ -d zfs-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/zfs-$rel.tar.gz | tar xz || true"
+  - (cd spl-$rel && ./configure --prefix=/usr && make && sudo make install) || true
   - (cd zfs-$rel && ./configure --prefix=/usr && make && sudo make install)
   - sudo modprobe zfs
   - cd $TRAVIS_BUILD_DIR
diff --git a/backup/backup.go b/backup/backup.go
index 5f0f0bb..e66156b 100644
--- a/backup/backup.go
+++ b/backup/backup.go
@@ -21,6 +21,7 @@
 package backup
 
 import (
+	"bytes"
 	"context"
 	"crypto/md5" //nolint:gosec // Not used for cryptography
 	"encoding/json"
@@ -477,10 +478,11 @@ func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInf
 	var group *errgroup.Group
 	group, ctx = errgroup.WithContext(ctx)
 
+	buf := bytes.NewBuffer(nil)
 	cmd := zfs.GetZFSSendCommand(ctx, j)
 	cin, cout := io.Pipe()
 	cmd.Stdout = cout
-	cmd.Stderr = os.Stderr
+	cmd.Stderr = buf
 	counter := datacounter.NewReaderCounter(cin)
 	usingPipe := false
 	if j.MaxFileBuffer == 0 {
@@ -591,7 +593,7 @@ func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInf
 
 	err = group.Wait()
 	if err != nil {
-		log.AppLogger.Errorf("Error waiting for zfs command to finish - %v", err)
+		log.AppLogger.Errorf("Error waiting for zfs command to finish - %v: %s", err, buf.String())
 		return err
 	}
 	log.AppLogger.Infof("zfs send completed without error")
diff --git a/backup/restore.go b/backup/restore.go
index ecd80e0..3c4c757 100644
--- a/backup/restore.go
+++ b/backup/restore.go
@@ -21,6 +21,7 @@
 package backup
 
 import (
+	"bytes"
 	"context"
 	"crypto/md5" // nolint:gosec // MD5 not used for cryptographic purposes here
 	"errors"
@@ -244,32 +245,21 @@ func Receive(pctx context.Context, jobInfo *files.JobInfo) error {
 		}
 	}
 
-	// Compute the Manifest File
-	tempManifest, err := files.CreateManifestVolume(ctx, jobInfo)
-	if err != nil {
-		log.AppLogger.Errorf("Error trying to create manifest volume - %v", err)
-		return err
-	}
-	if err = tempManifest.Close(); err != nil {
-		log.AppLogger.Warningf("Could not close temporary manifest %v", err)
-	}
-	if err = tempManifest.DeleteVolume(); err != nil {
-		log.AppLogger.Warningf("Could not delete temporary manifest %v", err)
-	}
+	manifestObjectName := jobInfo.ManifestObjectName()
 	// nolint:gosec // MD5 not used for cryptographic purposes here
-	safeManifestFile := fmt.Sprintf("%x", md5.Sum([]byte(tempManifest.ObjectName)))
+	safeManifestFile := fmt.Sprintf("%x", md5.Sum([]byte(manifestObjectName)))
 	safeManifestPath := filepath.Join(localCachePath, safeManifestFile)
 
 	// Check to see if we have the manifest file locally
 	manifest, err := readManifest(ctx, safeManifestPath, jobInfo)
 	if err != nil {
 		if os.IsNotExist(err) {
-			if bErr := backend.PreDownload(ctx, []string{tempManifest.ObjectName}); bErr != nil {
-				log.AppLogger.Errorf("Error trying to pre download manifest volume %s - %v", tempManifest.ObjectName, bErr)
+			if bErr := backend.PreDownload(ctx, []string{manifestObjectName}); bErr != nil {
+				log.AppLogger.Errorf("Error trying to pre download manifest volume %s - %v", manifestObjectName, bErr)
 				return bErr
 			}
 			// Try and download the manifest file from the backend
-			if dErr := downloadTo(ctx, backend, tempManifest.ObjectName, safeManifestPath); dErr != nil {
+			if dErr := downloadTo(ctx, backend, manifestObjectName, safeManifestPath); dErr != nil {
 				return dErr
 			}
 			manifest, err = readManifest(ctx, safeManifestPath, jobInfo)
@@ -460,9 +450,10 @@ func processSequence(ctx context.Context, sequence downloadSequence, backend bac
 }
 
 func receiveStream(ctx context.Context, cmd *exec.Cmd, j *files.JobInfo, c <-chan *files.VolumeInfo, buffer <-chan interface{}) error {
+	buf := bytes.NewBuffer(nil)
 	cin, cout := io.Pipe()
 	cmd.Stdin = cin
-	cmd.Stderr = os.Stderr
+	cmd.Stderr = buf
 	var group *errgroup.Group
 	var once sync.Once
 	group, ctx = errgroup.WithContext(ctx)
@@ -533,7 +524,7 @@ func receiveStream(ctx context.Context, cmd *exec.Cmd, j *files.JobInfo, c <-cha
 	// Wait for the command to finish
 	err = group.Wait()
 	if err != nil {
-		log.AppLogger.Errorf("Error waiting for zfs command to finish - %v", err)
+		log.AppLogger.Errorf("Error waiting for zfs command to finish - %v: %s", err, buf.String())
 		return err
 	}
 	log.AppLogger.Infof("zfs receive completed without error")
diff --git a/cmd/clean.go b/cmd/clean.go
index 9e5d256..8ceb4be 100644
--- a/cmd/clean.go
+++ b/cmd/clean.go
@@ -21,8 +21,6 @@
 package cmd
 
 import (
-	"context"
-
 	"github.com/spf13/cobra"
 
 	"github.com/someone1/zfsbackup-go/backup"
@@ -39,7 +37,7 @@ var cleanCmd = &cobra.Command{
 	PreRunE:       validateCleanFlags,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		jobInfo.Destinations = []string{args[0]}
-		return backup.Clean(context.Background(), &jobInfo, cleanLocal)
+		return backup.Clean(cmd.Context(), &jobInfo, cleanLocal)
 	},
 }
 
diff --git a/cmd/list.go b/cmd/list.go
index 2b51ebb..0fc54b0 100644
--- a/cmd/list.go
+++ b/cmd/list.go
@@ -21,7 +21,6 @@
 package cmd
 
 import (
-	"context"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -62,7 +61,7 @@ var listCmd = &cobra.Command{
 		}
 
 		jobInfo.Destinations = []string{args[0]}
-		return backup.List(context.Background(), &jobInfo, startsWith, before, after)
+		return backup.List(cmd.Context(), &jobInfo, startsWith, before, after)
 	},
 }
 
diff --git a/cmd/receive.go b/cmd/receive.go
index 7dcc768..c075be2 100644
--- a/cmd/receive.go
+++ b/cmd/receive.go
@@ -45,9 +45,9 @@ var receiveCmd = &cobra.Command{
 		log.AppLogger.Infof("Limiting the number of active files to %d", jobInfo.MaxFileBuffer)
 
 		if jobInfo.AutoRestore {
-			return backup.AutoRestore(context.Background(), &jobInfo)
+			return backup.AutoRestore(cmd.Context(), &jobInfo)
 		}
-		return backup.Receive(context.Background(), &jobInfo)
+		return backup.Receive(cmd.Context(), &jobInfo)
 	},
 }
 
diff --git a/cmd/root.go b/cmd/root.go
index ac1fec7..3e11026 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -21,6 +21,7 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -72,8 +73,8 @@ destination of your choosing.`,
 
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
-func Execute() {
-	if err := RootCmd.Execute(); err != nil {
+func Execute(ctx context.Context) {
+	if err := RootCmd.ExecuteContext(ctx); err != nil {
 		os.Exit(-1)
 	}
 }
@@ -243,9 +244,14 @@ func getAndDecryptPrivateKey(email string) (*openpgp.Entity, error) {
 }
 
 func loadSendKeys() error {
-	if jobInfo.EncryptTo != "" && publicKeyRingPath == "" {
-		log.AppLogger.Errorf("You must specify a public keyring path if you provide an encryptTo option")
-		return errInvalidInput
+	if jobInfo.EncryptTo != "" {
+		if usingSmartOption() && secretKeyRingPath == "" {
+			log.AppLogger.Errorf("You must specify a secret keyring path if you use a smart option with encryptTo")
+			return errInvalidInput
+		} else if publicKeyRingPath == "" {
+			log.AppLogger.Errorf("You must specify a public keyring path if you provide an encryptTo option")
+			return errInvalidInput
+		}
 	}
 
 	if jobInfo.SignFrom != "" && secretKeyRingPath == "" {
@@ -254,7 +260,13 @@ func loadSendKeys() error {
 	}
 
 	if jobInfo.EncryptTo != "" {
-		if jobInfo.EncryptKey = pgp.GetPublicKeyByEmail(jobInfo.EncryptTo); jobInfo.EncryptKey == nil {
+		if usingSmartOption() {
+			var err error
+			jobInfo.EncryptKey, err = getAndDecryptPrivateKey(jobInfo.EncryptTo)
+			if err != nil {
+				return err
+			}
+		} else if jobInfo.EncryptKey = pgp.GetPublicKeyByEmail(jobInfo.EncryptTo); jobInfo.EncryptKey == nil {
 			log.AppLogger.Errorf("Could not find public key for %s", jobInfo.EncryptTo)
 			return errInvalidInput
 		}
diff --git a/cmd/send.go b/cmd/send.go
index f2a07a5..cc6f78c 100644
--- a/cmd/send.go
+++ b/cmd/send.go
@@ -63,7 +63,7 @@ var sendCmd = &cobra.Command{
 			log.AppLogger.Infof("Will be signed from %s", jobInfo.SignFrom)
 		}
 
-		return backup.Backup(context.Background(), &jobInfo)
+		return backup.Backup(cmd.Context(), &jobInfo)
 	},
 }
 
@@ -309,12 +309,6 @@ func validateSendFlags(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if usingSmartOption() {
-		if err := loadReceiveKeys(); err != nil {
-			return err
-		}
-	}
-
 	if jobInfo.IncrementalSnapshot.Name != "" && fullIncremental != "" {
 		log.AppLogger.Errorf("The flags -i and -I are mutually exclusive. Please specify only one of these flags.")
 		return errInvalidInput
diff --git a/files/jobinfo.go b/files/jobinfo.go
index 8d27b93..ac982d3 100644
--- a/files/jobinfo.go
+++ b/files/jobinfo.go
@@ -202,3 +202,54 @@ func (j *JobInfo) ValidateSendFlags() error {
 
 	return nil
 }
+
+func (j *JobInfo) ManifestObjectName() string {
+	extensions := []string{"manifest"}
+	nameParts := []string{j.ManifestPrefix}
+
+	baseParts, ext := j.volumeNameParts(true)
+	extensions = append(extensions, ext...)
+	nameParts = append(nameParts, baseParts...)
+
+	return fmt.Sprintf("%s.%s", strings.Join(nameParts, j.Separator), strings.Join(extensions, "."))
+}
+
+func (j *JobInfo) BackupVolumeObjectName(volumeNumber int64) string {
+	extensions := []string{"zstream"}
+
+	nameParts, ext := j.volumeNameParts(false)
+	extensions = append(extensions, ext...)
+	extensions = append(extensions, fmt.Sprintf("vol%d", volumeNumber))
+
+	return fmt.Sprintf("%s.%s", strings.Join(nameParts, j.Separator), strings.Join(extensions, "."))
+}
+
+func (j *JobInfo) volumeNameParts(isManifest bool) (nameParts, extensions []string) {
+	extensions = make([]string, 0, 2)
+
+	if j.EncryptKey != nil || j.SignKey != nil {
+		extensions = append(extensions, "pgp")
+	}
+
+	compressorName := j.Compressor
+	if isManifest {
+		compressorName = InternalCompressor
+	}
+
+	switch compressorName {
+	case InternalCompressor:
+		extensions = append([]string{"gz"}, extensions...)
+	case "", ZfsCompressor:
+	default:
+		extensions = append([]string{compressorName}, extensions...)
+	}
+
+	nameParts = []string{j.VolumeName}
+	if j.IncrementalSnapshot.Name != "" {
+		nameParts = append(nameParts, j.IncrementalSnapshot.Name, "to", j.BaseSnapshot.Name)
+	} else {
+		nameParts = append(nameParts, j.BaseSnapshot.Name)
+	}
+
+	return nameParts, extensions
+}
diff --git a/files/volumeinfo.go b/files/volumeinfo.go
index 8f8ba06..108ea4b 100644
--- a/files/volumeinfo.go
+++ b/files/volumeinfo.go
@@ -23,6 +23,7 @@ package files
 import (
 	"bufio"
 	"context"
+	"crypto"
 	"crypto/md5"  // nolint:gosec // MD5 not used for cryptographic purposes here
 	"crypto/sha1" // nolint:gosec // SHA1 not used for cryptographic purposes here
 	"crypto/sha256"
@@ -406,26 +407,31 @@ func (v *VolumeInfo) CopyTo(dest string) (err error) {
 
 // prepareVolume returns a VolumeInfo, filename parts, extension parts, and an error
 // compress -> encrypt/sign -> output
-// nolint:gocritic // Don't need to name the results
-func prepareVolume(ctx context.Context, j *JobInfo, pipe, isManifest bool) (*VolumeInfo, []string, []string, error) {
+func prepareVolume(ctx context.Context, j *JobInfo, pipe, isManifest bool) (*VolumeInfo, error) {
 	v, err := CreateSimpleVolume(ctx, pipe)
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, err
 	}
 
-	extensions := make([]string, 0, 2)
-
 	// Prepare the Encryption/Signing writer, if required
 	if j.EncryptKey != nil || j.SignKey != nil {
-		extensions = append(extensions, "pgp")
 		pgpConfig := new(packet.Config)
 		pgpConfig.DefaultCompressionAlgo = packet.CompressionNone // We will do our own, thank you very much!
 		pgpConfig.DefaultCipher = packet.CipherAES256
+		pgpConfig.DefaultHash = crypto.SHA256
+		pgpConfig.RSABits = 2048
 		fileHints := new(openpgp.FileHints)
 		fileHints.IsBinary = true
-		pgpWriter, err := openpgp.Encrypt(v.w, []*openpgp.Entity{j.EncryptKey}, j.SignKey, fileHints, pgpConfig)
-		if err != nil {
-			return nil, nil, nil, err
+
+		var pgpWriter io.WriteCloser
+		if j.EncryptKey != nil {
+			if pgpWriter, err = openpgp.Encrypt(v.w, []*openpgp.Entity{j.EncryptKey}, j.SignKey, fileHints, pgpConfig); err != nil {
+				return nil, err
+			}
+		} else {
+			if pgpWriter, err = openpgp.Sign(v.w, j.SignKey, fileHints, pgpConfig); err != nil {
+				return nil, err
+			}
 		}
 		v.pgpw = pgpWriter
 		v.w = pgpWriter
@@ -441,7 +447,7 @@ func prepareVolume(ctx context.Context, j *JobInfo, pipe, isManifest bool) (*Vol
 	case InternalCompressor:
 		v.cw, _ = gzip.NewWriterLevel(v.w, j.CompressionLevel)
 		v.w = v.cw
-		extensions = append([]string{"gz"}, extensions...)
+
 		printCompressCMD.Do(func() {
 			log.AppLogger.Infof("Will be using internal gzip compressor with compression level %d.", j.CompressionLevel)
 		})
@@ -450,14 +456,12 @@ func prepareVolume(ctx context.Context, j *JobInfo, pipe, isManifest bool) (*Vol
 	case ZfsCompressor:
 		printCompressCMD.Do(func() { log.AppLogger.Infof("Will send a ZFS compressed stream") })
 	default:
-		extensions = append([]string{compressorName}, extensions...)
-
 		v.cmd = exec.CommandContext(ctx, compressorName, "-c", fmt.Sprintf("-%d", j.CompressionLevel))
 		v.cmd.Stdout = v.w
 
 		compressor, err := v.cmd.StdinPipe()
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, err
 		}
 		v.cw = compressor
 		v.w = v.cw
@@ -472,20 +476,13 @@ func prepareVolume(ctx context.Context, j *JobInfo, pipe, isManifest bool) (*Vol
 
 		err = v.cmd.Start()
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, err
 		}
 
 		// TODO: Signal properly if the process closes prematurely
 	}
 
-	nameParts := []string{j.VolumeName}
-	if j.IncrementalSnapshot.Name != "" {
-		nameParts = append(nameParts, j.IncrementalSnapshot.Name, "to", j.BaseSnapshot.Name)
-	} else {
-		nameParts = append(nameParts, j.BaseSnapshot.Name)
-	}
-
-	return v, nameParts, extensions, nil
+	return v, nil
 }
 
 // CreateManifestVolume will call CreateSimpleVolume and add options to compress,
@@ -493,18 +490,12 @@ func prepareVolume(ctx context.Context, j *JobInfo, pipe, isManifest bool) (*Vol
 // It will also name the file accordingly as a manifest file.
 func CreateManifestVolume(ctx context.Context, j *JobInfo) (*VolumeInfo, error) {
 	// Create and name the manifest file
-	extensions := []string{"manifest"}
-	nameParts := []string{j.ManifestPrefix}
-
-	v, baseParts, ext, err := prepareVolume(ctx, j, false, true)
+	v, err := prepareVolume(ctx, j, false, true)
 	if err != nil {
 		return nil, err
 	}
 
-	extensions = append(extensions, ext...)
-	nameParts = append(nameParts, baseParts...)
-
-	v.ObjectName = fmt.Sprintf("%s.%s", strings.Join(nameParts, j.Separator), strings.Join(extensions, "."))
+	v.ObjectName = j.ManifestObjectName()
 	v.IsManifest = true
 
 	return v, nil
@@ -514,24 +505,18 @@ func CreateManifestVolume(ctx context.Context, j *JobInfo) (*VolumeInfo, error)
 // encrypt, and/or sign the file as it is written depending on the provided options.
 // It will also name the file accordingly as a volume as part of backup set.
 func CreateBackupVolume(ctx context.Context, j *JobInfo, volnum int64) (*VolumeInfo, error) {
-	// Create and name the backup file
-	extensions := []string{"zstream"}
-
 	pipe := false
 	if j.MaxFileBuffer == 0 {
 		pipe = true
 	}
 
-	v, nameParts, ext, err := prepareVolume(ctx, j, pipe, false)
+	v, err := prepareVolume(ctx, j, pipe, false)
 	if err != nil {
 		return nil, err
 	}
 
 	v.VolumeNumber = volnum
-	extensions = append(extensions, ext...)
-	extensions = append(extensions, fmt.Sprintf("vol%d", v.VolumeNumber))
-
-	v.ObjectName = fmt.Sprintf("%s.%s", strings.Join(nameParts, j.Separator), strings.Join(extensions, "."))
+	v.ObjectName = j.BackupVolumeObjectName(v.VolumeNumber)
 
 	return v, nil
 }
diff --git a/integration_test.go b/integration_test.go
index f922f63..4169c43 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -26,6 +26,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	oglog "log"
 	"net/url"
 	"os"
 	"os/exec"
@@ -39,12 +40,14 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/op/go-logging"
 
 	"github.com/someone1/zfsbackup-go/backends"
 	"github.com/someone1/zfsbackup-go/backup"
 	"github.com/someone1/zfsbackup-go/cmd"
 	"github.com/someone1/zfsbackup-go/config"
 	"github.com/someone1/zfsbackup-go/files"
+	"github.com/someone1/zfsbackup-go/log"
 )
 
 const s3TestBucketName = "s3integrationbuckettest"
@@ -217,9 +220,12 @@ func compareDirs(t *testing.T, source, dest string) {
 	t.Helper()
 
 	// nolint:gosec // The input is safe
-	diffCmd := exec.Command("diff", "-rq", source, dest)
-	err := diffCmd.Run()
-	if err != nil {
+	diffCmd := exec.Command("diff", "-rq", "--exclude", ".zfs", source, dest)
+	errBuf := bytes.NewBuffer(nil)
+	diffCmd.Stderr = errBuf
+
+	if err := diffCmd.Run(); err != nil {
+		t.Logf("diff output: %s", errBuf.String())
 		t.Fatalf("unexpected difference comparing %s with %s: %v", source, dest, err)
 	}
 }
@@ -428,9 +434,12 @@ func restoreWrapper(bucket, target string) func(*testing.T) {
 	}
 }
 
-// TestEncryption expects private.pgp and public.pgp to be available with the test@example.com user
-func TestEncryption(t *testing.T) {
-	tempDir, err := ioutil.TempDir("", "pgptest")
+// TestEncryptionAndSign expects private.pgp and public.pgp to be available with the test@example.com user
+func TestEncryptionAndSign(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	tempDir, err := ioutil.TempDir("", t.Name())
 	if err != nil {
 		t.Fatalf("error preparing temp dir for tests - %v", err)
 	}
@@ -449,33 +458,106 @@ func TestEncryption(t *testing.T) {
 		newDataset = fmt.Sprintf("tank/%snew", t.Name())
 	)
 
-	copyDataset(t, "tank/data@a", dataset)
+	copyDataset(t, "tank/data@c", dataset)
 	defer deleteDataset(t, dataset)
 	defer deleteDataset(t, newDataset)
 
-	dataset += "@a"
-
-	cmd.ResetSendJobInfo()
-
-	// Manual Full Backup
-	cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--workingDirectory", scratchDir, "--publicKeyRingPath", "public.pgp", "--encryptTo", user, dataset, target})
-	if err := cmd.RootCmd.Execute(); err != nil {
-		t.Fatalf("error performing backup: %v", err)
+	tests := []struct {
+		name    string
+		args    []string
+		wantErr bool
+	}{
+		{
+			"Encrypted Backup - Fail no public keyring",
+			[]string{
+				"send", "--logLevel", logLevel, "--workingDirectory", scratchDir,
+				"--secretKeyRingPath", "private.pgp", "--encryptTo", user, fmt.Sprintf("%s@a", dataset), target,
+			},
+			true,
+		},
+		{
+			"Signed Backup - Fail no secret keyring",
+			[]string{
+				"send", "--logLevel", logLevel, "--workingDirectory", scratchDir,
+				"--publicKeyRingPath", "public.pgp", "--signFrom", user, fmt.Sprintf("%s@a", dataset), target,
+			},
+			true,
+		},
+		{
+			"Manual Full Backup - Encrypted",
+			[]string{
+				"send", "--logLevel", logLevel, "--workingDirectory", scratchDir,
+				"--publicKeyRingPath", "public.pgp", "--encryptTo", user, fmt.Sprintf("%s@a", dataset), target,
+			},
+			false,
+		},
+		{
+			"Manual Incremental Backup - Signed",
+			[]string{
+				"send", "--logLevel", logLevel, "--workingDirectory", scratchDir,
+				"--secretKeyRingPath", "private.pgp", "--signFrom", user, "-i", fmt.Sprintf("%s@a", dataset), fmt.Sprintf("%s@b", dataset), target,
+			},
+			false,
+		},
+		{
+			"Smart Encrypted Backup - Fail no secret keyring",
+			[]string{
+				"send", "--logLevel", logLevel, "--workingDirectory", scratchDir,
+				"--publicKeyRingPath", "public.pgp", "--encryptTo", user, "--increment", dataset, target,
+			},
+			true,
+		},
+		{
+			"Smart Encrypted & Signed Backup - Success",
+			[]string{
+				"send", "--logLevel", logLevel, "--workingDirectory", scratchDir,
+				"--publicKeyRingPath", "public.pgp", "--secretKeyRingPath", "private.pgp", "--encryptTo", user, "--signFrom", user, "--increment", dataset, target,
+			},
+			false,
+		},
+		{
+			"Restore Failure - No Key Ring",
+			[]string{"receive", "--logLevel", logLevel, "--workingDirectory", scratchDir, "-F", fmt.Sprintf("%s@a", dataset), target, newDataset},
+			true,
+		},
+		{
+			"Full Restore success - Encrypted",
+			[]string{"receive", "--logLevel", logLevel, "--workingDirectory", scratchDir, "--secretKeyRingPath", "private.pgp", "--encryptTo", user, "-F", fmt.Sprintf("%s@a", dataset), target, newDataset},
+			false,
+		},
+		{
+			"Incremental Restore success - Signed",
+			[]string{"receive", "--logLevel", logLevel, "--workingDirectory", scratchDir, "--publicKeyRingPath", "public.pgp", "--signFrom", user, "-F", "-i", fmt.Sprintf("%s@a", dataset), fmt.Sprintf("%s@b", dataset), target, newDataset},
+			false,
+		},
+		{
+			"Smart Restore success - Encrypted & Signed",
+			[]string{"receive", "--logLevel", logLevel, "--workingDirectory", scratchDir, "--publicKeyRingPath", "public.pgp", "--secretKeyRingPath", "private.pgp", "--encryptTo", user, "--signFrom", user, "-F", "--auto", dataset, target, newDataset},
+			false,
+		},
 	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			switch tt.args[0] {
+			case "send":
+				cmd.ResetSendJobInfo()
+			case "receive":
+				cmd.ResetReceiveJobInfo()
+			}
 
-	// Restore Failure
-	cmd.ResetReceiveJobInfo()
+			cmd.RootCmd.SetArgs(tt.args)
 
-	cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--workingDirectory", scratchDir, "-F", dataset, target, newDataset})
-	if err := cmd.RootCmd.Execute(); err == nil {
-		t.Fatalf("expected an error performing receive")
-	}
+			buf := bytes.NewBuffer(nil)
 
-	// Restore success
-	cmd.ResetReceiveJobInfo()
+			log.AppLogger.SetBackend(logging.MultiLogger(logging.NewLogBackend(buf, "", oglog.Ldate|oglog.Ltime)))
 
-	cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--workingDirectory", scratchDir, "--secretKeyRingPath", "private.pgp", "--encryptTo", user, "-F", dataset, target, newDataset})
-	if err := cmd.RootCmd.Execute(); err != nil {
-		t.Fatalf("error performing receive: %v", err)
+			if err := cmd.RootCmd.ExecuteContext(ctx); (err != nil) != tt.wantErr {
+				t.Logf("%s", buf.String())
+				t.Errorf("zfsbackup error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
 	}
+
+	compareDirs(t, "/tank/data", fmt.Sprintf("/%s", newDataset))
 }
diff --git a/main.go b/main.go
index 19d8e94..e13507d 100644
--- a/main.go
+++ b/main.go
@@ -21,9 +21,14 @@
 package main
 
 import (
+	"context"
+
 	"github.com/someone1/zfsbackup-go/cmd"
 )
 
 func main() {
-	cmd.Execute()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	cmd.Execute(ctx)
 }
diff --git a/travis-setup.sh b/travis-setup.sh
index 9028bb3..111da03 100755
--- a/travis-setup.sh
+++ b/travis-setup.sh
@@ -2,8 +2,9 @@
 
 # Setup ZFS Pool
 export VDEV=$(mktemp)
-dd if=/dev/zero of=${VDEV} bs=2048 count=1048576
+dd if=/dev/zero of=${VDEV} bs=2048 count=1572864
 sudo zpool create tank ${VDEV}
+sudo zfs set snapdir=visible compression=lz4 atime=off tank
 sudo zfs create tank/data
 sudo dd if=/dev/urandom of=/tank/data/a bs=1024 count=409600
 sudo zfs snapshot tank/data@a
diff --git a/travis-teardown.sh b/travis-teardown.sh
index 43c54e2..aec1c9d 100755
--- a/travis-teardown.sh
+++ b/travis-teardown.sh
@@ -12,7 +12,7 @@ sudo zpool destroy -f tank || true
 sudo rm ${VDEV} || true
 
 # Remove PGP keyrings
-rm *.pgp
+rm *.pgp || true
 
 # Clear env variables
 export VDEV=

From 6fba3e83faf645b159286a49def47f624db7fde7 Mon Sep 17 00:00:00 2001
From: Prateek Malhotra <someone1@gmail.com>
Date: Sun, 3 May 2020 02:48:20 -0400
Subject: [PATCH 2/7] chore: update go modules and travis config

---
 .travis.yml         |  4 ++--
 go.mod              | 20 +++++++++++---------
 go.sum              | 32 ++++++++++++++++++++++++++++++++
 integration_test.go | 29 +++++++++++++++++++----------
 4 files changed, 64 insertions(+), 21 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 8502d5f..fc7ca13 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -34,8 +34,8 @@ before_install:
   - sudo apt-get install -y linux-headers-`uname -r` tree uuid-dev libattr1-dev libblkid-dev jq gnupg2 xz-utils gzip
   - mkdir -p $HOME/zfs
   - cd $HOME/zfs
-  - "[[ -d spl-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/spl-$rel.tar.gz | tar xz"
-  - "[[ -d zfs-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/zfs-$rel.tar.gz | tar xz || true"
+  - "[[ -d spl-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/spl-$rel.tar.gz | tar xz || true"
+  - "[[ -d zfs-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/zfs-$rel.tar.gz | tar xz"
   - (cd spl-$rel && ./configure --prefix=/usr && make && sudo make install) || true
   - (cd zfs-$rel && ./configure --prefix=/usr && make && sudo make install)
   - sudo modprobe zfs
diff --git a/go.mod b/go.mod
index 08b9128..c62cff9 100644
--- a/go.mod
+++ b/go.mod
@@ -6,16 +6,17 @@ require (
 	cloud.google.com/go v0.56.0 // indirect
 	cloud.google.com/go/storage v1.6.0
 	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
-	github.com/Azure/azure-sdk-for-go v41.2.0+incompatible
+	github.com/Azure/azure-sdk-for-go v42.0.0+incompatible
 	github.com/Azure/azure-storage-blob-go v0.8.0
 	github.com/Azure/go-autorest/autorest v0.10.0 // indirect
 	github.com/Azure/go-autorest/autorest/to v0.3.0 // indirect
-	github.com/aws/aws-sdk-go v1.30.8
+	github.com/aws/aws-sdk-go v1.30.19
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/dnaeon/go-vcr v1.0.1 // indirect
 	github.com/dustin/go-humanize v1.0.0
+	github.com/golang/protobuf v1.4.0 // indirect
 	github.com/juju/ratelimit v1.0.1
-	github.com/klauspost/compress v1.10.4 // indirect
+	github.com/klauspost/compress v1.10.5 // indirect
 	github.com/klauspost/pgzip v1.2.3
 	github.com/kurin/blazer v0.5.3
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
@@ -26,11 +27,12 @@ require (
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/crypto v0.0.0-20200414173820-0848c9571904
+	golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
+	golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 // indirect
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
-	golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 // indirect
-	golang.org/x/tools v0.0.0-20200416214402-fc959738d646 // indirect
-	google.golang.org/api v0.21.0
-	google.golang.org/genproto v0.0.0-20200413115906-b5235f65be36 // indirect
-	google.golang.org/grpc v1.28.1 // indirect
+	golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 // indirect
+	golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770 // indirect
+	google.golang.org/api v0.22.0
+	google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84 // indirect
+	google.golang.org/grpc v1.29.1 // indirect
 )
diff --git a/go.sum b/go.sum
index 33ccbd9..6ad4f2a 100644
--- a/go.sum
+++ b/go.sum
@@ -33,6 +33,8 @@ github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot
 github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
 github.com/Azure/azure-sdk-for-go v41.2.0+incompatible h1:JOlv1wDuxcJi1ExJpQLNfEj6znsTFt2TiwQMow2YaXI=
 github.com/Azure/azure-sdk-for-go v41.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go v42.0.0+incompatible h1:yz6sFf5bHZ+gEOQVuK5JhPqTTAmv+OvSLSaqgzqaCwY=
+github.com/Azure/azure-sdk-for-go v42.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
 github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
@@ -63,6 +65,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.30.8 h1:4BHbh8K3qKmcnAgToZ2LShldRF9inoqIBccpCLNCy3I=
 github.com/aws/aws-sdk-go v1.30.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.30.19 h1:vRwsYgbUvC25Cb3oKXTyTYk3R5n1LRVk8zbvL4inWsc=
+github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
@@ -124,6 +128,12 @@ github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
 github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -165,6 +175,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.4 h1:jFzIFaf586tquEB5EhzQG0HwGNSlgAJpG53G6Ss11wc=
 github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqjewrc=
+github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/pgzip v1.2.3 h1:Ce2to9wvs/cuJ2b86/CKQoTYr9VHfpanYosZ0UBJqdw=
 github.com/klauspost/pgzip v1.2.3/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -258,6 +270,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 h1:bXoxMPcSLOq08zI3/c5dEBT6lE4eh+jOh886GHrn6V8=
 golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -313,6 +327,8 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -353,6 +369,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 h1:opSr2sbRXk5X5/givKrrKj9HXxFpW2sdCiP8MJSKLQY=
 golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 h1:5B6i6EAiSYyejWfvc5Rc9BbI3rzIsrrXfAQBWnYfn+w=
+golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -395,6 +413,8 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200416214402-fc959738d646 h1:7CEkhBsBejkW845gR1AmglqMfc1yGzn42FBmtM4jxyM=
 golang.org/x/tools v0.0.0-20200416214402-fc959738d646/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770 h1:M9Fif0OxNji8w+HvmhVQ8KJtiZOsjU9RgslJGhn95XE=
+golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -410,6 +430,8 @@ google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
 google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.21.0 h1:zS+Q/CJJnVlXpXQVIz+lH0ZT2lBuT2ac7XD8Y/3w6hY=
 google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0 h1:J1Pl9P2lnmYFSJvgs70DKELqHNh8CNWXPbud4njEE2s=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -437,6 +459,8 @@ google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200413115906-b5235f65be36 h1:j7CmVRD4Kec0+f8VuBAc2Ak2MFfXm5Q2/RxuJLL+76E=
 google.golang.org/genproto v0.0.0-20200413115906-b5235f65be36/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84 h1:pSLkPbrjnPyLDYUO2VM9mDLqo2V6CFBY84lFSZAfoi4=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -450,6 +474,14 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k=
 google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
diff --git a/integration_test.go b/integration_test.go
index 4169c43..6147791 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -231,6 +231,9 @@ func compareDirs(t *testing.T, source, dest string) {
 }
 
 func TestIntegration(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	removeAzureBucket := setupAzureBucket(t)
 	defer removeAzureBucket()
 
@@ -248,7 +251,7 @@ func TestIntegration(t *testing.T) {
 
 		// Manual Full Backup
 		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "tank/data@a", bucket})
-		if err := cmd.RootCmd.Execute(); err != nil {
+		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing backup: %v", err)
 		}
 
@@ -265,7 +268,7 @@ func TestIntegration(t *testing.T) {
 
 		// Manual Incremental Backup from bookmark
 		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "-i", "tank/data#a", "tank/data@b", bucket})
-		if err := cmd.RootCmd.Execute(); err != nil {
+		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing backup: %v", err)
 		}
 
@@ -282,7 +285,7 @@ func TestIntegration(t *testing.T) {
 
 		// "Smart" incremental Backup from bookmark
 		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "--compressor", "xz", "--compressionLevel", "2", "--increment", "tank/data", bucket})
-		if err := cmd.RootCmd.Execute(); err != nil {
+		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing backup: %v", err)
 		}
 
@@ -290,7 +293,7 @@ func TestIntegration(t *testing.T) {
 
 		// Smart Incremental Backup - Nothing to do
 		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "--increment", "tank/data", bucket})
-		if err := cmd.RootCmd.Execute(); err != backup.ErrNoOp {
+		if err := cmd.RootCmd.ExecuteContext(ctx); err != backup.ErrNoOp {
 			t.Fatalf("expecting error %v, but got %v instead", backup.ErrNoOp, err)
 		}
 
@@ -300,7 +303,7 @@ func TestIntegration(t *testing.T) {
 	// Restore tank/data for tests
 	deleteDataset(t, "tank/data")
 	cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "--auto", "tank/data", s3bucket, "tank/data"})
-	if err := cmd.RootCmd.Execute(); err != nil {
+	if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 		t.Fatalf("error performing receive: %v", err)
 	}
 
@@ -320,6 +323,9 @@ func TestIntegration(t *testing.T) {
 
 func listWrapper(bucket string) func(*testing.T) {
 	return func(t *testing.T) {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
 		old := config.Stdout
 		buf := bytes.NewBuffer(nil)
 		config.Stdout = buf
@@ -361,7 +367,7 @@ func listWrapper(bucket string) func(*testing.T) {
 			cmd.ResetListJobInfo()
 
 			cmd.RootCmd.SetArgs(append(opts, bucket))
-			if err := cmd.RootCmd.Execute(); err != nil {
+			if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 				t.Fatalf("error performing backup: %v", err)
 			}
 
@@ -387,6 +393,9 @@ func listWrapper(bucket string) func(*testing.T) {
 
 func restoreWrapper(bucket, target string) func(*testing.T) {
 	return func(t *testing.T) {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
 		scratchDir, sErr := ioutil.TempDir("", "")
 		if sErr != nil {
 			t.Fatalf("could not create temp scratch dir: %v", sErr)
@@ -398,7 +407,7 @@ func restoreWrapper(bucket, target string) func(*testing.T) {
 
 		// Restore to snapshot @a (full)
 		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "-F", "tank/data@a", bucket, target})
-		if err := cmd.RootCmd.Execute(); err != nil {
+		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing receive: %v", err)
 		}
 
@@ -406,7 +415,7 @@ func restoreWrapper(bucket, target string) func(*testing.T) {
 
 		// Restore to snapshot @b from @a (incremental)
 		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "-F", "-i", "tank/data@a", "tank/data@b", bucket, target})
-		if err := cmd.RootCmd.Execute(); err != nil {
+		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing receive: %v", err)
 		}
 
@@ -414,7 +423,7 @@ func restoreWrapper(bucket, target string) func(*testing.T) {
 
 		// Restore to latest snapshot @c (auto)
 		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "--workingDirectory", scratchDir, "-F", "--auto", "tank/data", bucket, target})
-		if err := cmd.RootCmd.Execute(); err != nil {
+		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing receive: %v", err)
 		}
 
@@ -424,7 +433,7 @@ func restoreWrapper(bucket, target string) func(*testing.T) {
 
 		// Restore to snapshot @c from origin tank/data@b (auto)
 		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "-F", "--auto", "-o", "origin=tank/data@b", "tank/data", bucket, target + "origin"})
-		if err := cmd.RootCmd.Execute(); err != nil {
+		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing receive: %v", err)
 		}
 

From 3a6aab1d0134333e9ea0d11c87aa82650e6dc204 Mon Sep 17 00:00:00 2001
From: Prateek Malhotra <someone1@gmail.com>
Date: Sun, 3 May 2020 20:58:07 -0400
Subject: [PATCH 3/7] fixup tests

---
 .travis.yml         |  4 ++-
 integration_test.go | 84 +++++++++++++++++++++++++++------------------
 travis-teardown.sh  |  4 +--
 3 files changed, 54 insertions(+), 38 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index fc7ca13..cd9c2c8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,6 +40,8 @@ before_install:
   - (cd zfs-$rel && ./configure --prefix=/usr && make && sudo make install)
   - sudo modprobe zfs
   - cd $TRAVIS_BUILD_DIR
+  - mkdir temp
+  - export TMPDIR=$PWD/temp
   - source ./travis-setup.sh
 
 install:
@@ -51,7 +53,7 @@ install:
 script:
   - make build
   - chmod +x ./integration_test.sh && ./integration_test.sh
-  - sudo -E $(which go) test -race -v -coverprofile=coverage.out -covermode=atomic -coverpkg=$(go list ./... | grep -v '/vendor/' | paste -sd, -) ./...  
+  - sudo -E TMPDIR=$TMPDIR $(which go) test -race -v -coverprofile=coverage.out -covermode=atomic -coverpkg=$(go list ./... | grep -v '/vendor/' | paste -sd, -) ./...  
   - make lint
 
 after_success:
diff --git a/integration_test.go b/integration_test.go
index 6147791..2e70cfc 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -183,6 +183,10 @@ func copyDataset(t *testing.T, source, dest string) {
 	// nolint:gosec // The input is safe
 	sendCMD := exec.Command("zfs", "send", "-R", source)
 	receiveCMD := exec.Command("zfs", "receive", dest)
+	sendBuf := bytes.NewBuffer(nil)
+	recBuf := bytes.NewBuffer(nil)
+	sendCMD.Stderr = sendBuf
+	receiveCMD.Stderr = recBuf
 
 	var err error
 	sendCMD.Stdout, err = receiveCMD.StdinPipe()
@@ -196,11 +200,11 @@ func copyDataset(t *testing.T, source, dest string) {
 	}()
 
 	if sErr := sendCMD.Run(); sErr != nil {
-		t.Fatalf("unexpected error sending dataset %s to %s - %v", source, dest, sErr)
+		t.Fatalf("unexpected error sending dataset %s to %s - %v: %s", source, dest, sErr, sendBuf.String())
 	}
 
 	if err = <-errChan; err != nil {
-		t.Fatalf("unexpected error receiving dataset %s to %s - %v", source, dest, err)
+		t.Fatalf("unexpected error receiving dataset %s to %s - %v: %s", source, dest, err, recBuf.String())
 	}
 }
 
@@ -210,9 +214,11 @@ func deleteDataset(t *testing.T, name string) {
 
 	// nolint:gosec // The input is safe
 	destroyCmd := exec.Command("zfs", "destroy", "-f", "-r", name)
+	destroyBuf := bytes.NewBuffer(nil)
+	destroyCmd.Stderr = destroyBuf
 
 	if err := destroyCmd.Run(); err != nil {
-		t.Fatalf("unexpected error deleting dataset %s - %v", name, err)
+		t.Fatalf("unexpected error deleting dataset %s - %v: %s", name, err, destroyBuf.String())
 	}
 }
 
@@ -249,6 +255,9 @@ func TestIntegration(t *testing.T) {
 	t.Run("Backup", func(t *testing.T) {
 		cmd.ResetSendJobInfo()
 
+		logBuf := bytes.NewBuffer(nil)
+		log.AppLogger.SetBackend(logging.MultiLogger(logging.NewLogBackend(logBuf, "", oglog.Ldate|oglog.Ltime)))
+
 		// Manual Full Backup
 		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "tank/data@a", bucket})
 		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
@@ -323,8 +332,10 @@ func TestIntegration(t *testing.T) {
 
 func listWrapper(bucket string) func(*testing.T) {
 	return func(t *testing.T) {
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
+		ctx := context.Background()
+
+		logBuf := bytes.NewBuffer(nil)
+		log.AppLogger.SetBackend(logging.MultiLogger(logging.NewLogBackend(logBuf, "", oglog.Ldate|oglog.Ltime)))
 
 		old := config.Stdout
 		buf := bytes.NewBuffer(nil)
@@ -353,48 +364,53 @@ func listWrapper(bucket string) func(*testing.T) {
 		}
 
 		for _, test := range listTests {
-			opts := []string{"list", "--logLevel", logLevel, "--jsonOutput"}
-			if test.volumeName != "" {
-				opts = append(opts, "--volumeName", test.volumeName)
-			}
-			if !test.after.IsZero() {
-				opts = append(opts, "--after", test.after.Format(time.RFC3339[:19]))
-			}
-			if !test.before.IsZero() {
-				opts = append(opts, "--before", test.before.Format(time.RFC3339[:19]))
-			}
+			test := test
+			t.Run(test.volumeName, func(t *testing.T) {
+				opts := []string{"list", "--logLevel", logLevel, "--jsonOutput"}
+				if test.volumeName != "" {
+					opts = append(opts, "--volumeName", test.volumeName)
+				}
+				if !test.after.IsZero() {
+					opts = append(opts, "--after", test.after.Format(time.RFC3339[:19]))
+				}
+				if !test.before.IsZero() {
+					opts = append(opts, "--before", test.before.Format(time.RFC3339[:19]))
+				}
 
-			cmd.ResetListJobInfo()
+				cmd.ResetListJobInfo()
 
-			cmd.RootCmd.SetArgs(append(opts, bucket))
-			if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
-				t.Fatalf("error performing backup: %v", err)
-			}
+				cmd.RootCmd.SetArgs(append(opts, bucket))
+				if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
+					t.Fatalf("error performing backup: %v", err)
+				}
 
-			jout := make(map[string][]*files.JobInfo)
-			if err := json.Unmarshal(buf.Bytes(), &jout); err != nil {
-				t.Fatalf("error parsing json output: %v", err)
-			}
+				jout := make(map[string][]*files.JobInfo)
+				if err := json.Unmarshal(buf.Bytes(), &jout); err != nil {
+					t.Fatalf("error parsing json output: %v", err)
+				}
 
-			if len(jout) != test.keys || len(jout["tank/data"]) != test.entries {
-				t.Fatalf("expected %d keys and %d entries, got %d keys and %d entries", test.keys, test.entries, len(jout), len(jout["tank/data"]))
-			}
+				if len(jout) != test.keys || len(jout["tank/data"]) != test.entries {
+					t.Fatalf("expected %d keys and %d entries, got %d keys and %d entries", test.keys, test.entries, len(jout), len(jout["tank/data"]))
+				}
 
-			if len(jout["tank/data"]) == 3 {
-				if jout["tank/data"][0].BaseSnapshot.Name != "a" || jout["tank/data"][1].BaseSnapshot.Name != "b" || jout["tank/data"][2].BaseSnapshot.Name != "c" {
-					t.Fatalf("expected snapshot order a -> b -> c, but got %s -> %s -> %s instead", jout["tank/data"][0].BaseSnapshot.Name, jout["tank/data"][1].BaseSnapshot.Name, jout["tank/data"][2].BaseSnapshot.Name)
+				if len(jout["tank/data"]) == 3 {
+					if jout["tank/data"][0].BaseSnapshot.Name != "a" || jout["tank/data"][1].BaseSnapshot.Name != "b" || jout["tank/data"][2].BaseSnapshot.Name != "c" {
+						t.Fatalf("expected snapshot order a -> b -> c, but got %s -> %s -> %s instead", jout["tank/data"][0].BaseSnapshot.Name, jout["tank/data"][1].BaseSnapshot.Name, jout["tank/data"][2].BaseSnapshot.Name)
+					}
 				}
-			}
 
-			buf.Reset()
+				buf.Reset()
+			})
 		}
 	}
 }
 
 func restoreWrapper(bucket, target string) func(*testing.T) {
 	return func(t *testing.T) {
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
+		ctx := context.Background()
+
+		logBuf := bytes.NewBuffer(nil)
+		log.AppLogger.SetBackend(logging.MultiLogger(logging.NewLogBackend(logBuf, "", oglog.Ldate|oglog.Ltime)))
 
 		scratchDir, sErr := ioutil.TempDir("", "")
 		if sErr != nil {
diff --git a/travis-teardown.sh b/travis-teardown.sh
index aec1c9d..87dd79a 100755
--- a/travis-teardown.sh
+++ b/travis-teardown.sh
@@ -15,6 +15,4 @@ sudo rm ${VDEV} || true
 rm *.pgp || true
 
 # Clear env variables
-export VDEV=
-export AWS_ACCESS_KEY_ID=
-export AWS_SECRET_ACCESS_KEY=
+unset VDEV

From 0b7a5c719575c2f7ea3212f9c34e8d4bdae39cd8 Mon Sep 17 00:00:00 2001
From: Prateek Malhotra <someone1@gmail.com>
Date: Sun, 3 May 2020 22:06:15 -0400
Subject: [PATCH 4/7] fixup test context

---
 backup/sync.go      |  1 +
 integration_test.go | 75 +++++++++++++++++++++++----------------------
 zfs/zfs.go          |  2 +-
 3 files changed, 40 insertions(+), 38 deletions(-)

diff --git a/backup/sync.go b/backup/sync.go
index 4497ae0..e9a80a5 100644
--- a/backup/sync.go
+++ b/backup/sync.go
@@ -139,6 +139,7 @@ func syncCache(ctx context.Context, j *files.JobInfo, localCache string, backend
 func validateSnapShotExists(ctx context.Context, snapshot *files.SnapshotInfo, target string, includeBookmarks bool) (bool, error) {
 	snapshots, err := zfs.GetSnapshotsAndBookmarks(ctx, target)
 	if err != nil {
+		log.AppLogger.Debugf("Could not list snapshots for %s: %v", target, err)
 		// TODO: There are some error cases that are ok to ignore!
 		return false, nil
 	}
diff --git a/integration_test.go b/integration_test.go
index 2e70cfc..6865aa2 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -180,6 +180,8 @@ func TestVersion(t *testing.T) {
 func copyDataset(t *testing.T, source, dest string) {
 	t.Helper()
 
+	t.Logf("Copying %s to %s", source, dest)
+
 	// nolint:gosec // The input is safe
 	sendCMD := exec.Command("zfs", "send", "-R", source)
 	receiveCMD := exec.Command("zfs", "receive", dest)
@@ -237,8 +239,7 @@ func compareDirs(t *testing.T, source, dest string) {
 }
 
 func TestIntegration(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	ctx := context.Background()
 
 	removeAzureBucket := setupAzureBucket(t)
 	defer removeAzureBucket()
@@ -249,6 +250,10 @@ func TestIntegration(t *testing.T) {
 	s3bucket := fmt.Sprintf("%s://%s", backends.AWSS3BackendPrefix, s3TestBucketName)
 	azurebucket := fmt.Sprintf("%s://%s", backends.AzureBackendPrefix, azureTestBucketName)
 	bucket := fmt.Sprintf("%s,%s", s3bucket, azurebucket)
+	dataset := fmt.Sprintf("tank/%s", t.Name())
+
+	copyDataset(t, "tank/data@c", dataset)
+	defer deleteDataset(t, dataset)
 
 	// Azurite doesn't seem to like '|' so making separator '-'
 	// Backup Tests
@@ -259,7 +264,7 @@ func TestIntegration(t *testing.T) {
 		log.AppLogger.SetBackend(logging.MultiLogger(logging.NewLogBackend(logBuf, "", oglog.Ldate|oglog.Ltime)))
 
 		// Manual Full Backup
-		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "tank/data@a", bucket})
+		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", fmt.Sprintf("%s@a", dataset), bucket})
 		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing backup: %v", err)
 		}
@@ -267,16 +272,18 @@ func TestIntegration(t *testing.T) {
 		cmd.ResetSendJobInfo()
 
 		// Bookmark setup
-		if err := exec.Command("zfs", "bookmark", "tank/data@a", "tank/data#a").Run(); err != nil {
-			t.Fatalf("unexpected error creating bookmark tank/data#a: %v", err)
+		// nolint:gosec // The input is safe
+		if err := exec.Command("zfs", "bookmark", fmt.Sprintf("%s@a", dataset), fmt.Sprintf("%s#a", dataset)).Run(); err != nil {
+			t.Fatalf("unexpected error creating bookmark %s#a: %v", dataset, err)
 		}
 
-		if err := exec.Command("zfs", "destroy", "tank/data@a").Run(); err != nil {
-			t.Fatalf("unexpected error destroying snapshot tank/data@a: %v", err)
+		// nolint:gosec // The input is safe
+		if err := exec.Command("zfs", "destroy", fmt.Sprintf("%s@a", dataset)).Run(); err != nil {
+			t.Fatalf("unexpected error destroying snapshot %s@a: %v", dataset, err)
 		}
 
 		// Manual Incremental Backup from bookmark
-		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "-i", "tank/data#a", "tank/data@b", bucket})
+		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "-i", fmt.Sprintf("%s#a", dataset), fmt.Sprintf("%s@b", dataset), bucket})
 		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing backup: %v", err)
 		}
@@ -284,16 +291,18 @@ func TestIntegration(t *testing.T) {
 		cmd.ResetSendJobInfo()
 
 		// Another Bookmark setup
-		if err := exec.Command("zfs", "bookmark", "tank/data@b", "tank/data#b").Run(); err != nil {
-			t.Fatalf("unexpected error creating bookmark tank/data#b: %v", err)
+		// nolint:gosec // The input is safe
+		if err := exec.Command("zfs", "bookmark", fmt.Sprintf("%s@b", dataset), fmt.Sprintf("%s#b", dataset)).Run(); err != nil {
+			t.Fatalf("unexpected error creating bookmark %s#b: %v", dataset, err)
 		}
 
-		if err := exec.Command("zfs", "destroy", "tank/data@b").Run(); err != nil {
-			t.Fatalf("unexpected error destroying snapshot tank/data@b: %v", err)
+		// nolint:gosec // The input is safe
+		if err := exec.Command("zfs", "destroy", fmt.Sprintf("%s@b", dataset)).Run(); err != nil {
+			t.Fatalf("unexpected error destroying snapshot %s@b: %v", dataset, err)
 		}
 
 		// "Smart" incremental Backup from bookmark
-		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "--compressor", "xz", "--compressionLevel", "2", "--increment", "tank/data", bucket})
+		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "--compressor", "xz", "--compressionLevel", "2", "--increment", dataset, bucket})
 		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing backup: %v", err)
 		}
@@ -301,7 +310,7 @@ func TestIntegration(t *testing.T) {
 		cmd.ResetSendJobInfo()
 
 		// Smart Incremental Backup - Nothing to do
-		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "--increment", "tank/data", bucket})
+		cmd.RootCmd.SetArgs([]string{"send", "--logLevel", logLevel, "--separator", "+", "--increment", dataset, bucket})
 		if err := cmd.RootCmd.ExecuteContext(ctx); err != backup.ErrNoOp {
 			t.Fatalf("expecting error %v, but got %v instead", backup.ErrNoOp, err)
 		}
@@ -309,13 +318,6 @@ func TestIntegration(t *testing.T) {
 		cmd.ResetSendJobInfo()
 	})
 
-	// Restore tank/data for tests
-	deleteDataset(t, "tank/data")
-	cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "--auto", "tank/data", s3bucket, "tank/data"})
-	if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
-		t.Fatalf("error performing receive: %v", err)
-	}
-
 	var restoreTest = []struct {
 		backend string
 		bucket  string
@@ -325,12 +327,12 @@ func TestIntegration(t *testing.T) {
 		{"Azure", azurebucket, "tank/data2"},
 	}
 	for _, test := range restoreTest {
-		t.Run(fmt.Sprintf("List%s", test.backend), listWrapper(test.bucket))
-		t.Run(fmt.Sprintf("Restore%s", test.backend), restoreWrapper(test.bucket, test.target))
+		t.Run(fmt.Sprintf("List%s", test.backend), listWrapper(dataset, test.bucket))
+		t.Run(fmt.Sprintf("Restore%s", test.backend), restoreWrapper(dataset, test.bucket, test.target))
 	}
 }
 
-func listWrapper(bucket string) func(*testing.T) {
+func listWrapper(dataset, bucket string) func(*testing.T) {
 	return func(t *testing.T) {
 		ctx := context.Background()
 
@@ -353,7 +355,7 @@ func listWrapper(bucket string) func(*testing.T) {
 			{"", time.Time{}, time.Time{}, 1, 3},
 			{"t*", time.Time{}, time.Time{}, 1, 3},
 			{"v*", time.Time{}, time.Time{}, 0, 0},
-			{"tank/data", time.Time{}, time.Time{}, 1, 3},
+			{dataset, time.Time{}, time.Time{}, 1, 3},
 			{"tan", time.Time{}, time.Time{}, 0, 0},
 			// before Tests
 			{"", time.Time{}, time.Now(), 1, 3},
@@ -389,13 +391,13 @@ func listWrapper(bucket string) func(*testing.T) {
 					t.Fatalf("error parsing json output: %v", err)
 				}
 
-				if len(jout) != test.keys || len(jout["tank/data"]) != test.entries {
-					t.Fatalf("expected %d keys and %d entries, got %d keys and %d entries", test.keys, test.entries, len(jout), len(jout["tank/data"]))
+				if len(jout) != test.keys || len(jout[dataset]) != test.entries {
+					t.Fatalf("expected %d keys and %d entries, got %d keys and %d entries", test.keys, test.entries, len(jout), len(jout[dataset]))
 				}
 
-				if len(jout["tank/data"]) == 3 {
-					if jout["tank/data"][0].BaseSnapshot.Name != "a" || jout["tank/data"][1].BaseSnapshot.Name != "b" || jout["tank/data"][2].BaseSnapshot.Name != "c" {
-						t.Fatalf("expected snapshot order a -> b -> c, but got %s -> %s -> %s instead", jout["tank/data"][0].BaseSnapshot.Name, jout["tank/data"][1].BaseSnapshot.Name, jout["tank/data"][2].BaseSnapshot.Name)
+				if len(jout[dataset]) == 3 {
+					if jout[dataset][0].BaseSnapshot.Name != "a" || jout[dataset][1].BaseSnapshot.Name != "b" || jout[dataset][2].BaseSnapshot.Name != "c" {
+						t.Fatalf("expected snapshot order a -> b -> c, but got %s -> %s -> %s instead", jout[dataset][0].BaseSnapshot.Name, jout[dataset][1].BaseSnapshot.Name, jout[dataset][2].BaseSnapshot.Name)
 					}
 				}
 
@@ -405,7 +407,7 @@ func listWrapper(bucket string) func(*testing.T) {
 	}
 }
 
-func restoreWrapper(bucket, target string) func(*testing.T) {
+func restoreWrapper(dataset, bucket, target string) func(*testing.T) {
 	return func(t *testing.T) {
 		ctx := context.Background()
 
@@ -422,7 +424,7 @@ func restoreWrapper(bucket, target string) func(*testing.T) {
 		cmd.ResetReceiveJobInfo()
 
 		// Restore to snapshot @a (full)
-		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "-F", "tank/data@a", bucket, target})
+		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "-F", fmt.Sprintf("%s@a", dataset), bucket, target})
 		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing receive: %v", err)
 		}
@@ -430,7 +432,7 @@ func restoreWrapper(bucket, target string) func(*testing.T) {
 		cmd.ResetReceiveJobInfo()
 
 		// Restore to snapshot @b from @a (incremental)
-		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "-F", "-i", "tank/data@a", "tank/data@b", bucket, target})
+		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "-F", "-i", fmt.Sprintf("%s@a", dataset), fmt.Sprintf("%s@b", dataset), bucket, target})
 		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing receive: %v", err)
 		}
@@ -438,7 +440,7 @@ func restoreWrapper(bucket, target string) func(*testing.T) {
 		cmd.ResetReceiveJobInfo()
 
 		// Restore to latest snapshot @c (auto)
-		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "--workingDirectory", scratchDir, "-F", "--auto", "tank/data", bucket, target})
+		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "--workingDirectory", scratchDir, "-F", "--auto", dataset, bucket, target})
 		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing receive: %v", err)
 		}
@@ -448,7 +450,7 @@ func restoreWrapper(bucket, target string) func(*testing.T) {
 		cmd.ResetReceiveJobInfo()
 
 		// Restore to snapshot @c from origin tank/data@b (auto)
-		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "-F", "--auto", "-o", "origin=tank/data@b", "tank/data", bucket, target + "origin"})
+		cmd.RootCmd.SetArgs([]string{"receive", "--logLevel", logLevel, "--separator", "+", "-F", "--auto", "-o", "origin=tank/data@b", dataset, bucket, target + "origin"})
 		if err := cmd.RootCmd.ExecuteContext(ctx); err != nil {
 			t.Fatalf("error performing receive: %v", err)
 		}
@@ -461,8 +463,7 @@ func restoreWrapper(bucket, target string) func(*testing.T) {
 
 // TestEncryptionAndSign expects private.pgp and public.pgp to be available with the test@example.com user
 func TestEncryptionAndSign(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	ctx := context.Background()
 
 	tempDir, err := ioutil.TempDir("", t.Name())
 	if err != nil {
diff --git a/zfs/zfs.go b/zfs/zfs.go
index b90eef5..3d21f08 100644
--- a/zfs/zfs.go
+++ b/zfs/zfs.go
@@ -88,7 +88,7 @@ func GetSnapshotsAndBookmarks(ctx context.Context, target string) ([]files.Snaps
 	}
 	err = cmd.Wait()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%s (%v)", strings.TrimSpace(errB.String()), err)
 	}
 
 	return snapshots, nil

From 844a4be1844526f8ef81bd01e0324b7e4e420d08 Mon Sep 17 00:00:00 2001
From: Prateek Malhotra <someone1@gmail.com>
Date: Mon, 25 May 2020 17:10:43 -0400
Subject: [PATCH 5/7] getting zfs to work in travis is hard

---
 .travis.yml         | 5 +++--
 integration_test.sh | 4 ++--
 travis-setup.sh     | 3 +--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index cd9c2c8..433d7a1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,7 +42,8 @@ before_install:
   - cd $TRAVIS_BUILD_DIR
   - mkdir temp
   - export TMPDIR=$PWD/temp
-  - source ./travis-setup.sh
+  - export VDEV=$(mktemp)
+  - chmod +x ./travis-setup.sh && ./travis-setup.sh
 
 install:
   - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $GOPATH/bin v1.24.0
@@ -60,4 +61,4 @@ after_success:
   - sudo -E $GOPATH/bin/goveralls -coverprofile=coverage.out -service=travis-ci
 
 after_script: 
-  - source ./travis-teardown.sh
+  - chmod +x ./travis-teardown.sh && ./travis-teardown.sh
diff --git a/integration_test.sh b/integration_test.sh
index 38ef89b..3083504 100755
--- a/integration_test.sh
+++ b/integration_test.sh
@@ -3,8 +3,8 @@
 SCRATCHDIR=$(mktemp -d)
 mkdir -p $SCRATCHDIR/integrationtest
 
-export TARGET="file://$SCRATCHDIR/integrationtest/"
-export USER="test@example.com"
+TARGET="file://$SCRATCHDIR/integrationtest/"
+USER="test@example.com"
 
 echo "Basic send/recieve test with encryption and compression"
 go build ./
diff --git a/travis-setup.sh b/travis-setup.sh
index 111da03..d7e0f50 100755
--- a/travis-setup.sh
+++ b/travis-setup.sh
@@ -1,9 +1,8 @@
 #!/bin/sh -ev
 
 # Setup ZFS Pool
-export VDEV=$(mktemp)
 dd if=/dev/zero of=${VDEV} bs=2048 count=1572864
-sudo zpool create tank ${VDEV}
+sudo zpool create -f tank ${VDEV}
 sudo zfs set snapdir=visible compression=lz4 atime=off tank
 sudo zfs create tank/data
 sudo dd if=/dev/urandom of=/tank/data/a bs=1024 count=409600

From 66271626d49dc13eeb97919142700704a1ecc07b Mon Sep 17 00:00:00 2001
From: Prateek Malhotra <someone1@gmail.com>
Date: Mon, 25 May 2020 19:05:04 -0400
Subject: [PATCH 6/7] disabled zfs 0.8 from tests

---
 .travis.yml     | 2 +-
 travis-setup.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 433d7a1..bb74cc2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,7 +24,7 @@ env:
     - GCS_FAKE_SERVER="https://localhost:4443"
   matrix:
     - rel=0.7.13
-    - rel=0.8.3
+    # - rel=0.8.3 # some feature preventing zpool creation (encryption)?
 
 before_install:
   - export MAKEFLAGS=-j$(($(grep -c '^processor' /proc/cpuinfo) * 2 + 1))
diff --git a/travis-setup.sh b/travis-setup.sh
index d7e0f50..9aef00a 100755
--- a/travis-setup.sh
+++ b/travis-setup.sh
@@ -2,7 +2,7 @@
 
 # Setup ZFS Pool
 dd if=/dev/zero of=${VDEV} bs=2048 count=1572864
-sudo zpool create -f tank ${VDEV}
+sudo zpool create tank ${VDEV}
 sudo zfs set snapdir=visible compression=lz4 atime=off tank
 sudo zfs create tank/data
 sudo dd if=/dev/urandom of=/tank/data/a bs=1024 count=409600

From 91b8fa08041e64f5a1994220b4df408fc4ff7bbe Mon Sep 17 00:00:00 2001
From: Prateek Malhotra <someone1@gmail.com>
Date: Mon, 25 May 2020 19:42:33 -0400
Subject: [PATCH 7/7] chore: update go deps

---
 go.mod | 23 ++++++---------
 go.sum | 88 +++++++++++++++++++++++++++++++++++-----------------------
 2 files changed, 62 insertions(+), 49 deletions(-)

diff --git a/go.mod b/go.mod
index c62cff9..7ee8d4f 100644
--- a/go.mod
+++ b/go.mod
@@ -3,21 +3,19 @@ module github.com/someone1/zfsbackup-go
 go 1.13
 
 require (
-	cloud.google.com/go v0.56.0 // indirect
-	cloud.google.com/go/storage v1.6.0
+	cloud.google.com/go/storage v1.8.0
 	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
-	github.com/Azure/azure-sdk-for-go v42.0.0+incompatible
+	github.com/Azure/azure-sdk-for-go v42.3.0+incompatible
 	github.com/Azure/azure-storage-blob-go v0.8.0
-	github.com/Azure/go-autorest/autorest v0.10.0 // indirect
+	github.com/Azure/go-autorest/autorest v0.10.2 // indirect
 	github.com/Azure/go-autorest/autorest/to v0.3.0 // indirect
-	github.com/aws/aws-sdk-go v1.30.19
+	github.com/aws/aws-sdk-go v1.31.4
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/dnaeon/go-vcr v1.0.1 // indirect
 	github.com/dustin/go-humanize v1.0.0
-	github.com/golang/protobuf v1.4.0 // indirect
 	github.com/juju/ratelimit v1.0.1
-	github.com/klauspost/compress v1.10.5 // indirect
-	github.com/klauspost/pgzip v1.2.3
+	github.com/klauspost/compress v1.10.6 // indirect
+	github.com/klauspost/pgzip v1.2.4
 	github.com/kurin/blazer v0.5.3
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/miolini/datacounter v1.0.2
@@ -27,12 +25,7 @@ require (
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
-	golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 // indirect
+	golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
-	golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 // indirect
-	golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770 // indirect
-	google.golang.org/api v0.22.0
-	google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84 // indirect
-	google.golang.org/grpc v1.29.1 // indirect
+	google.golang.org/api v0.25.0
 )
diff --git a/go.sum b/go.sum
index 6ad4f2a..e4d64c8 100644
--- a/go.sum
+++ b/go.sum
@@ -9,12 +9,18 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T
 cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
 cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
 cloud.google.com/go v0.56.0 h1:WRz29PgAsVEyPSDHyk+0fpEkwEFyfhHn+JbksT6gIL4=
 cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0 h1:a/O/bK/vWrYGOTFtH8di4rBxMZnmkjy+Y5LxpDwo+dA=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
@@ -22,24 +28,26 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0 h1:86K1Gel7BQ9/WmNWn7dTKMvTLFzwtBe5FNqYbi9X35g=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
 github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY=
 github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
-github.com/Azure/azure-sdk-for-go v41.2.0+incompatible h1:JOlv1wDuxcJi1ExJpQLNfEj6znsTFt2TiwQMow2YaXI=
-github.com/Azure/azure-sdk-for-go v41.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/azure-sdk-for-go v42.0.0+incompatible h1:yz6sFf5bHZ+gEOQVuK5JhPqTTAmv+OvSLSaqgzqaCwY=
-github.com/Azure/azure-sdk-for-go v42.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go v42.3.0+incompatible h1:PAHkmPqd/vQV4LJcqzEUM1elCyTMWjbrO8oFMl0dvBE=
+github.com/Azure/azure-sdk-for-go v42.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
 github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.10.0 h1:mvdtztBqcL8se7MdrUweNieTNi4kfNG6GOJuurQJpuY=
-github.com/Azure/go-autorest/autorest v0.10.0/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
+github.com/Azure/go-autorest/autorest v0.10.2 h1:NuSF3gXetiHyUbVdneJMEVyPUYAe5wh+aN08JYAf1tI=
+github.com/Azure/go-autorest/autorest v0.10.2/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
 github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0=
 github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
@@ -63,10 +71,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/aws/aws-sdk-go v1.30.8 h1:4BHbh8K3qKmcnAgToZ2LShldRF9inoqIBccpCLNCy3I=
-github.com/aws/aws-sdk-go v1.30.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.30.19 h1:vRwsYgbUvC25Cb3oKXTyTYk3R5n1LRVk8zbvL4inWsc=
-github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.31.4 h1:YZ0uEYIWeanGuAomElHmRWMAbXVqrQixxgf2vtIjO6M=
+github.com/aws/aws-sdk-go v1.31.4/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
@@ -120,12 +126,14 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
 github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
@@ -134,6 +142,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -148,6 +158,7 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
@@ -173,12 +184,10 @@ github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSg
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.10.4 h1:jFzIFaf586tquEB5EhzQG0HwGNSlgAJpG53G6Ss11wc=
-github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqjewrc=
-github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/pgzip v1.2.3 h1:Ce2to9wvs/cuJ2b86/CKQoTYr9VHfpanYosZ0UBJqdw=
-github.com/klauspost/pgzip v1.2.3/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/klauspost/compress v1.10.6 h1:SP6zavvTG3YjOosWePXFDlExpKIWMTO4SE/Y8MZB2vI=
+github.com/klauspost/compress v1.10.6/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
+github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -268,10 +277,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 h1:bXoxMPcSLOq08zI3/c5dEBT6lE4eh+jOh886GHrn6V8=
-golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88=
-golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw=
+golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -325,10 +332,13 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 h1:MsuvTghUPjX762sGLnGsxC3HM0B5r83wEtYcYR8/vRs=
 golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA=
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f h1:QBjCr1Fz5kw158VqdE9JfI9cJnl/ymnJWAdMuinqL7Y=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -365,12 +375,12 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 h1:opSr2sbRXk5X5/givKrrKj9HXxFpW2sdCiP8MJSKLQY=
-golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 h1:5B6i6EAiSYyejWfvc5Rc9BbI3rzIsrrXfAQBWnYfn+w=
-golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25 h1:OKbAoGs4fGM5cPLlVQLZGYkFC8OnOfgo6tt0Smf9XhM=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -410,11 +420,13 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2 h1:L/G4KZvrQn7FWLN/LlulBtBzrLUhqjiGfTWWDmrh+IQ=
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200416214402-fc959738d646 h1:7CEkhBsBejkW845gR1AmglqMfc1yGzn42FBmtM4jxyM=
-golang.org/x/tools v0.0.0-20200416214402-fc959738d646/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770 h1:M9Fif0OxNji8w+HvmhVQ8KJtiZOsjU9RgslJGhn95XE=
-golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88 h1:4j84u0sokprDu3IdSYHJMmou+YSLflMz8p7yAx/QI4g=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -427,17 +439,22 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.21.0 h1:zS+Q/CJJnVlXpXQVIz+lH0ZT2lBuT2ac7XD8Y/3w6hY=
-google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.22.0 h1:J1Pl9P2lnmYFSJvgs70DKELqHNh8CNWXPbud4njEE2s=
 google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0 h1:cG03eaksBzhfSIk7JRGctfp3lanklcOM/mTGvow7BbQ=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.25.0 h1:LodzhlzZEUfhXzNUMIfVlf9Gr6Ua5MMtoFWh7+f47qA=
+google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -456,11 +473,14 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 h1:YzfoEYWbODU5Fbt37+h7X16BWQbad7Q4S6gclTKFXM8=
 google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200413115906-b5235f65be36 h1:j7CmVRD4Kec0+f8VuBAc2Ak2MFfXm5Q2/RxuJLL+76E=
-google.golang.org/genproto v0.0.0-20200413115906-b5235f65be36/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84 h1:pSLkPbrjnPyLDYUO2VM9mDLqo2V6CFBY84lFSZAfoi4=
 google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380 h1:xriR1EgvKfkKxIoU2uUvrMVl+H26359loFFUleSMXFo=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -472,8 +492,6 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k=
-google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -482,6 +500,8 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0 h1:cJv5/xdbk1NnMPR1VP9+HU6gupuG9MLBoH1r6RHZ2MY=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=