Create public compression package for compression enumeration; disabl…

…e "none" compression for tarball layer

internal/compression/compression.go

@@ -21,79 +21,75 @@ import (
 
 	"github.com/google/go-containerregistry/internal/gzip"
 	"github.com/google/go-containerregistry/internal/zstd"
-)
-
-type Compression string
-
-// The collection of known MediaType values.
-const (
-	None Compression = "none"
-	GZip Compression = "gzip"
-	ZStd Compression = "zstd"
+	"github.com/google/go-containerregistry/pkg/compression"
 )
 
 type Opener = func() (io.ReadCloser, error)
 
-func GetCompression(opener Opener) (Compression, error) {
+// GetCompression detects whether an Opener is compressed and which algorithm is used.
+func GetCompression(opener Opener) (compression.Compression, error) {
 	rc, err := opener()
 	if err != nil {
-		return None, err
+		return compression.None, err
 	}
 	defer rc.Close()
 
-	compression, _, err := PeekCompression(rc)
+	cp, _, err := PeekCompression(rc)
 	if err != nil {
-		return None, err
+		return compression.None, err
 	}
 
-	return compression, nil
-}
-
-// PeekReader is an io.Reader that also implements Peek a la bufio.Reader.
-type PeekReader interface {
-	io.Reader
-	Peek(n int) ([]byte, error)
+	return cp, nil
 }
 
 // PeekCompression detects whether the input stream is compressed and which algorithm is used.
 //
 // If r implements Peek, we will use that directly, otherwise a small number
-// of bytes are buffered to Peek at the gzip header, and the returned
+// of bytes are buffered to Peek at the gzip/zstd header, and the returned
 // PeekReader can be used as a replacement for the consumed input io.Reader.
-func PeekCompression(r io.Reader) (Compression, PeekReader, error) {
-	var pr PeekReader
-	if p, ok := r.(PeekReader); ok {
-		pr = p
-	} else {
-		pr = bufio.NewReader(r)
-	}
+func PeekCompression(r io.Reader) (compression.Compression, PeekReader, error) {
+	pr := intoPeekReader(r)
 
-	var header []byte
-	var err error
+	if isGZip, _, err := checkHeader(pr, gzip.MagicHeader); err != nil {
+		return compression.None, pr, err
+	} else if isGZip {
+		return compression.GZip, pr, nil
+	}
 
-	if header, err = pr.Peek(2); err != nil {
-		// https://github.com/google/go-containerregistry/issues/367
-		if err == io.EOF {
-			return None, pr, nil
-		}
-		return None, pr, err
+	if isZStd, _, err := checkHeader(pr, zstd.MagicHeader); err != nil {
+		return compression.None, pr, err
+	} else if isZStd {
+		return compression.ZStd, pr, nil
 	}
 
-	if bytes.Equal(header, gzip.MagicHeader) {
-		return GZip, pr, nil
+	return compression.None, pr, nil
+}
+
+// PeekReader is an io.Reader that also implements Peek a la bufio.Reader.
+type PeekReader interface {
+	io.Reader
+	Peek(n int) ([]byte, error)
+}
+
+// IntoPeekReader creates a PeekReader from an io.Reader.
+// If the reader already has a Peek method, it will just return the passed reader.
+func intoPeekReader(r io.Reader) PeekReader {
+	if p, ok := r.(PeekReader); ok {
+		return p
+	} else {
+		return bufio.NewReader(r)
 	}
+}
 
-	if header, err = pr.Peek(4); err != nil {
+// CheckHeader checks whether the first bytes from a PeekReader match an expected header
+func checkHeader(pr PeekReader, expectedHeader []byte) (bool, PeekReader, error) {
+	header, err := pr.Peek(len(expectedHeader))
+	if err != nil {
 		// https://github.com/google/go-containerregistry/issues/367
 		if err == io.EOF {
-			return None, pr, nil
+			return false, pr, nil
 		}
-		return None, pr, err
+		return false, pr, err
 	}
-
-	if bytes.Equal(header, zstd.MagicHeader) {
-		return ZStd, pr, nil
-	}
-
-	return None, pr, nil
+	return bytes.Equal(header, expectedHeader), pr, nil
 }

internal/compression/compression_test.go

@@ -0,0 +1,76 @@
+// Copyright 2020 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compression
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/google/go-containerregistry/internal/and"
+	"github.com/google/go-containerregistry/internal/gzip"
+	"github.com/google/go-containerregistry/internal/zstd"
+)
+
+type Compressor = func(rc io.ReadCloser) io.ReadCloser
+type Decompressor = func(rc io.ReadCloser) (io.ReadCloser, error)
+
+func testPeekCompression(t *testing.T,
+	compressionExpected string,
+	compress Compressor,
+	decompress Decompressor,
+) {
+	content := "This is the input string."
+	contentBuf := bytes.NewBufferString(content)
+
+	compressed := compress(io.NopCloser(contentBuf))
+	compressionDetected, pr, err := PeekCompression(compressed)
+
+	if err != nil {
+		t.Error("PeekCompression() =", err)
+	}
+
+	if got := string(compressionDetected); got != compressionExpected {
+		t.Errorf("PeekCompression(); got %q, content %q", got, compressionExpected)
+	}
+
+	decompressed, err := decompress(withCloser(pr, compressed))
+
+	b, err := io.ReadAll(decompressed)
+	if err != nil {
+		t.Error("ReadAll() =", err)
+	}
+
+	if got := string(b); got != content {
+		t.Errorf("ReadAll(); got %q, content %q", got, content)
+	}
+}
+
+func TestPeekCompression(t *testing.T) {
+	testPeekCompression(t, "gzip", gzip.ReadCloser, gzip.UnzipReadCloser)
+	testPeekCompression(t, "zstd", zstd.ReadCloser, zstd.UnzipReadCloser)
+
+	nopCompress := func(rc io.ReadCloser) io.ReadCloser { return rc }
+	nopDecompress := func(rc io.ReadCloser) (io.ReadCloser, error) { return rc, nil }
+
+	testPeekCompression(t, "none", nopCompress, nopDecompress)
+}
+
+func withCloser(pr PeekReader, rc io.ReadCloser) io.ReadCloser {
+	return &and.ReadCloser{
+		Reader:    pr,
+		CloseFunc: rc.Close,
+	}
+}

pkg/compression/compression.go

@@ -0,0 +1,25 @@
+// Copyright 2022 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compression
+
+// Compression is an enumeration of the supported compression algorithms
+type Compression string
+
+// The collection of known MediaType values.
+const (
+	None Compression = "none"
+	GZip Compression = "gzip"
+	ZStd Compression = "zstd"
+)

pkg/v1/partial/compressed.go

@@ -21,6 +21,7 @@ import (
 	"github.com/google/go-containerregistry/internal/compression"
 	"github.com/google/go-containerregistry/internal/gzip"
 	"github.com/google/go-containerregistry/internal/zstd"
+	comp "github.com/google/go-containerregistry/pkg/compression"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/types"
 )
@@ -54,21 +55,22 @@ func (cle *compressedLayerExtender) Uncompressed() (io.ReadCloser, error) {
 	}
 
 	// Often, the "compressed" bytes are not actually-compressed.
-	// Peek at the first two bytes to determine whether or not it's correct to
+	// Peek at the first two bytes to determine whether it's correct to
 	// wrap this with gzip.UnzipReadCloser or zstd.UnzipReadCloser.
 	cp, pr, err := compression.PeekCompression(rc)
 	if err != nil {
 		return nil, err
 	}
+
 	prc := &and.ReadCloser{
 		Reader:    pr,
 		CloseFunc: rc.Close,
 	}
 
 	switch cp {
-	case compression.GZip:
+	case comp.GZip:
 		return gzip.UnzipReadCloser(prc)
-	case compression.ZStd:
+	case comp.ZStd:
 		return zstd.UnzipReadCloser(prc)
 	default:
 		return prc, nil

pkg/v1/tarball/image.go

@@ -27,7 +27,8 @@ import (
 	"path/filepath"
 	"sync"
 
-	"github.com/google/go-containerregistry/internal/compression"
+	comp "github.com/google/go-containerregistry/internal/compression"
+	"github.com/google/go-containerregistry/pkg/compression"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
@@ -167,7 +168,7 @@ func (i *image) areLayersCompressed() (bool, error) {
 	}
 	defer blob.Close()
 
-	cp, _, err := compression.PeekCompression(blob)
+	cp, _, err := comp.PeekCompression(blob)
 	if err != nil {
 		return false, err
 	}

pkg/v1/tarball/layer.go

@@ -28,27 +28,19 @@ import (
 	gestargz "github.com/google/go-containerregistry/internal/estargz"
 	ggzip "github.com/google/go-containerregistry/internal/gzip"
 	"github.com/google/go-containerregistry/internal/zstd"
+	"github.com/google/go-containerregistry/pkg/compression"
+	"github.com/google/go-containerregistry/pkg/logs"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/types"
 )
 
-// LayerCompression is an enumeration of the supported compression algorithms for tarball layers
-type LayerCompression comp.Compression
-
-// The collection of known MediaType values.
-const (
-	None LayerCompression = "none"
-	GZip LayerCompression = "gzip"
-	ZStd LayerCompression = "zstd"
-)
-
 type layer struct {
 	digest             v1.Hash
 	diffID             v1.Hash
 	size               int64
 	compressedopener   Opener
 	uncompressedopener Opener
-	compression        LayerCompression
+	compression        compression.Compression
 	compressionLevel   int
 	annotations        map[string]string
 	estgzopts          []estargz.Option
@@ -104,16 +96,20 @@ type LayerOption func(*layer)
 
 // WithCompression is a functional option for overriding the default
 // compression algorithm used for compressing uncompressed tarballs.
-// Also updates the mediaType to "application/vnd.oci.image.layer.v1.tar+zstd"
+// Also sets the mediaType to "application/vnd.oci.image.layer.v1.tar+zstd"
 // if zstd compression is selected.
-func WithCompression(compression LayerCompression) LayerOption {
+func WithCompression(comp compression.Compression) LayerOption {
 	return func(l *layer) {
-		if compression == ZStd {
+		switch comp {
+		case compression.ZStd:
 			l.mediaType = types.OCILayerZStd
-		} else {
-			l.mediaType = types.DockerLayer
+		case compression.None:
+			logs.Warn.Printf("Compression type 'none' is not supported for tarball layers; using gzip compression.")
+		default:
+			logs.Warn.Printf("Unexpected compression type for WithCompression(): %s; using gzip compression instead.", comp)
 		}
-		l.compression = compression
+
+		l.compression = comp
 	}
 }
 
@@ -126,6 +122,7 @@ func WithCompressionLevel(level int) LayerOption {
 }
 
 // WithMediaType is a functional option for overriding the layer's media type.
+// Note that WithCompression overrides the mediaType if zstd compression is selected.
 func WithMediaType(mt types.MediaType) LayerOption {
 	return func(l *layer) {
 		l.mediaType = mt
@@ -227,13 +224,13 @@ func LayerFromFile(path string, opts ...LayerOption) (v1.Layer, error) {
 // Since gzip can be expensive, we support an option to memoize the
 // compression that can be passed here: tarball.WithCompressedCaching
 func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
-	compression, err := comp.GetCompression(opener)
+	comp, err := comp.GetCompression(opener)
 	if err != nil {
 		return nil, err
 	}
 
 	layer := &layer{
-		compression:      GZip,
+		compression:      compression.GZip,
 		compressionLevel: gzip.BestSpeed,
 		annotations:      make(map[string]string, 1),
 		mediaType:        types.DockerLayer,
@@ -243,8 +240,8 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
 		opts = append([]LayerOption{WithEstargz}, opts...)
 	}
 
-	switch compression {
-	case comp.GZip:
+	switch comp {
+	case compression.GZip:
 		layer.compressedopener = opener
 		layer.uncompressedopener = func() (io.ReadCloser, error) {
 			urc, err := opener()
@@ -253,7 +250,7 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
 			}
 			return ggzip.UnzipReadCloser(urc)
 		}
-	case comp.ZStd:
+	case compression.ZStd:
 		layer.compressedopener = opener
 		layer.uncompressedopener = func() (io.ReadCloser, error) {
 			urc, err := opener()
@@ -270,7 +267,7 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
 				return nil, err
 			}
 
-			if layer.compression == ZStd {
+			if layer.compression == compression.ZStd {
 				return zstd.ReadCloserLevel(crc, layer.compressionLevel), nil
 			}