stream snapshot to FSM when restoring from archive

The `RestoreFromArchive` helper decompresses the snapshot archive to a
temporary file before reading it into the FSM. For large snapshots
this performs a lot of disk IO. Stream decompress the snapshot as we
read it, without first writing to a temporary file.

helper/raftutil/snapshot.go

@@ -3,13 +3,12 @@ package raftutil
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
-	"os"
 
 	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/raft"
+
 	"github.com/hashicorp/nomad/helper/snapshot"
 	"github.com/hashicorp/nomad/nomad/state"
-	"github.com/hashicorp/raft"
 )
 
 func RestoreFromArchive(archive io.Reader) (*state.StateStore, *raft.SnapshotMeta, error) {
@@ -20,27 +19,31 @@ func RestoreFromArchive(archive io.Reader) (*state.StateStore, *raft.SnapshotMet
 		return nil, nil, fmt.Errorf("failed to create FSM: %w", err)
 	}
 
-	snap, err := ioutil.TempFile("", "snap-")
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to create a temp file: %w", err)
-	}
-	defer os.Remove(snap.Name())
-	defer snap.Close()
+	// r is closed by Restore, w is closed by CopySnapshot
+	r, w := io.Pipe()
 
-	meta, err := snapshot.CopySnapshot(archive, snap)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to read snapshot: %w", err)
-	}
+	errCh := make(chan error)
+	metaCh := make(chan *raft.SnapshotMeta)
 
-	_, err = snap.Seek(0, 0)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to seek: %w", err)
-	}
+	go func() {
+		meta, err := snapshot.CopySnapshot(archive, w)
+		if err != nil {
+			errCh <- fmt.Errorf("failed to read snapshot: %w", err)
+		}
+		if meta != nil {
+			metaCh <- meta
+		}
+	}()
 
-	err = fsm.Restore(snap)
+	err = fsm.Restore(r)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to restore from snapshot: %w", err)
 	}
 
-	return fsm.State(), meta, nil
+	select {
+	case err := <-errCh:
+		return nil, nil, err
+	case meta := <-metaCh:
+		return fsm.State(), meta, nil
+	}
 }

helper/snapshot/snapshot.go

@@ -138,13 +138,22 @@ func (s *Snapshot) Close() error {
 	return os.Remove(s.file.Name())
 }
 
+type Discard struct {
+	io.Writer
+}
+
+func (dc Discard) Close() error { return nil }
+
 // Verify takes the snapshot from the reader and verifies its contents.
 func Verify(in io.Reader) (*raft.SnapshotMeta, error) {
-	return CopySnapshot(in, ioutil.Discard)
+	return CopySnapshot(in, Discard{Writer: io.Discard})
 }
 
-// CopySnapshot copies the snapshot content from snapshot archive to dest
-func CopySnapshot(in io.Reader, dest io.Writer) (*raft.SnapshotMeta, error) {
+// CopySnapshot copies the snapshot content from snapshot archive to dest.
+// It will close the destination once complete.
+func CopySnapshot(in io.Reader, dest io.WriteCloser) (*raft.SnapshotMeta, error) {
+	defer dest.Close()
+
 	// Wrap the reader in a gzip decompressor.
 	decomp, err := gzip.NewReader(in)
 	if err != nil {