compression: add support for the zstd algorithm

zstd is a compression algorithm that has a very fast decoder, while
providing also good compression ratios.  The fast decoder makes it
suitable for container images, as decompressing the tarballs is a very
expensive operation.

opencontainers/image-spec#788 added support
for zstd to the OCI image specs.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

archive/compression/compression.go

@@ -29,6 +29,7 @@ import (
 	"sync"
 
 	"github.com/containerd/containerd/log"
+	"github.com/klauspost/compress/zstd"
 )
 
 type (
@@ -41,6 +42,8 @@ const (
 	Uncompressed Compression = iota
 	// Gzip is gzip compression algorithm.
 	Gzip
+	// Zstd is zstd compression algorithm.
+	Zstd
 )
 
 const disablePigzEnv = "CONTAINERD_DISABLE_PIGZ"
@@ -126,6 +129,7 @@ func (r *bufferedReader) Peek(n int) ([]byte, error) {
 func DetectCompression(source []byte) Compression {
 	for compression, m := range map[Compression][]byte{
 		Gzip: {0x1F, 0x8B, 0x08},
+		Zstd: {0x28, 0xb5, 0x2f, 0xfd},
 	} {
 		if len(source) < len(m) {
 			// Len too short
@@ -174,6 +178,19 @@ func DecompressStream(archive io.Reader) (DecompressReadCloser, error) {
 				return gzReader.Close()
 			},
 		}, nil
+	case Zstd:
+		zstdReader, err := zstd.NewReader(buf)
+		if err != nil {
+			return nil, err
+		}
+		return &readCloserWrapper{
+			Reader:      zstdReader,
+			compression: compression,
+			closer: func() error {
+				zstdReader.Close()
+				return nil
+			},
+		}, nil
 
 	default:
 		return nil, fmt.Errorf("unsupported compression format %s", (&compression).Extension())
@@ -187,6 +204,8 @@ func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, er
 		return &writeCloserWrapper{dest, nil}, nil
 	case Gzip:
 		return gzip.NewWriter(dest), nil
+	case Zstd:
+		return zstd.NewWriter(dest)
 	default:
 		return nil, fmt.Errorf("unsupported compression format %s", (&compression).Extension())
 	}
@@ -197,6 +216,8 @@ func (compression *Compression) Extension() string {
 	switch *compression {
 	case Gzip:
 		return "gz"
+	case Zstd:
+		return "zst"
 	}
 	return ""
 }

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/imdario/mergo v0.3.10
+	github.com/klauspost/compress v1.11.3
 	github.com/moby/sys/symlink v0.1.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1

go.sum

@@ -315,6 +315,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc=
+github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=

images/mediatypes.go

@@ -78,6 +78,8 @@ func DiffCompression(ctx context.Context, mediaType string) (string, error) {
 			switch ext[len(ext)-1] {
 			case "gzip":
 				return "gzip", nil
+			case "zstd":
+				return "zstd", nil
 			}
 		}
 		return "", nil