Add v1util.EstargzReadCloser utility. (#870)

* Add v1util.EstargzReadCloser utility.

This adds the first utility function we will need to start producing estargz layers.

* Move to internal package in anticipation of v1util explosion

* Clarify the usage of the library

* Update to latest estargz version (to get level option)

* Drop v1util

* Rename files for Jon

* Update estargz, drop multierror

* Add issue TODO

pkg/v1/internal/estargz/estargz.go

@@ -0,0 +1,51 @@
+// Copyright 2020 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package estargz
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+
+	"github.com/containerd/stargz-snapshotter/estargz"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+)
+
+// ReadCloser reads uncompressed tarball input from the io.ReadCloser and
+// returns:
+//  * An io.ReadCloser from which compressed data may be read, and
+//  * A v1.Hash with the hash of the estargz table of contents, or
+//  * An error if the estargz processing encountered a problem.
+//
+// Refer to estargz for the options:
+// https://pkg.go.dev/github.com/containerd/stargz-snapshotter@v0.2.0/estargz#Option
+func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (io.ReadCloser, v1.Hash, error) {
+	defer r.Close()
+
+	// TODO(#876): Avoid buffering into memory.
+	bs, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, v1.Hash{}, err
+	}
+	br := bytes.NewReader(bs)
+
+	rc, err := estargz.Build(io.NewSectionReader(br, 0, int64(len(bs))), opts...)
+	if err != nil {
+		return nil, v1.Hash{}, err
+	}
+
+	h, err := v1.NewHash(rc.TOCDigest().String())
+	return rc, h, err
+}

pkg/v1/internal/estargz/estargz_test.go

@@ -0,0 +1,109 @@
+// Copyright 2020 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package estargz
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/google/go-containerregistry/pkg/v1/internal/gzip"
+)
+
+func TestReader(t *testing.T) {
+	want := "This is the input string."
+	buf := bytes.NewBuffer(nil)
+	tw := tar.NewWriter(buf)
+
+	if err := tw.WriteHeader(&tar.Header{
+		Name: "foo",
+		Size: int64(len(want)),
+	}); err != nil {
+		t.Fatal("WriteHeader() =", err)
+	}
+	if _, err := tw.Write([]byte(want)); err != nil {
+		t.Fatal("tw.Write() =", err)
+	}
+	tw.Close()
+
+	zipped, _, err := ReadCloser(ioutil.NopCloser(buf))
+	if err != nil {
+		t.Fatal("ReadCloser() =", err)
+	}
+	unzipped, err := gzip.UnzipReadCloser(zipped)
+	if err != nil {
+		t.Error("gzip.UnzipReadCloser() =", err)
+	}
+	defer unzipped.Close()
+
+	found := false
+
+	r := tar.NewReader(unzipped)
+	for {
+		hdr, err := r.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			t.Fatal("tar.Next() =", err)
+		}
+
+		if hdr.Name != "foo" {
+			continue
+		}
+		found = true
+
+		b, err := ioutil.ReadAll(r)
+		if err != nil {
+			t.Error("ReadAll() =", err)
+		}
+		if got := string(b); got != want {
+			t.Errorf("ReadAll(); got %q, want %q", got, want)
+		}
+		if err := unzipped.Close(); err != nil {
+			t.Error("Close() =", err)
+		}
+	}
+
+	if !found {
+		t.Error(`Did not find the expected file "foo"`)
+	}
+}
+
+var (
+	errRead = fmt.Errorf("Read failed")
+)
+
+type failReader struct{}
+
+func (f failReader) Read(_ []byte) (int, error) {
+	return 0, errRead
+}
+
+func TestReadErrors(t *testing.T) {
+	fr := failReader{}
+
+	if _, _, err := ReadCloser(ioutil.NopCloser(fr)); err != errRead {
+		t.Error("ReadCloser: expected errRead, got", err)
+	}
+
+	buf := bytes.NewBufferString("not a tarball")
+	if _, _, err := ReadCloser(ioutil.NopCloser(buf)); !strings.Contains(err.Error(), "failed to parse tar file") {
+		t.Error(`ReadCloser: expected "failed to parse tar file", got`, err)
+	}
+}