add http zstd reader for cdi

Signed-off-by: huangzy <huangzynn@outlook.com>
kubevirt · Feb 21, 2023 · 4407d10 · 4407d10
1 parent f37088f
commit 4407d10
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 0 deletions.
diff --git a/pkg/image/filefmt.go b/pkg/image/filefmt.go
@@ -25,6 +25,12 @@ var knownHeaders = Headers{
 		SizeOff: 0,
 		SizeLen: 0,
 	},
+	"zst": Header{
+		Format:      "zst",
+		magicNumber: []byte{0x28, 0xb5, 0x2f, 0xfd},
+		SizeOff:     0,
+		SizeLen:     0,
+	},
 	"qcow2": Header{
 		Format:      "qcow2",
 		magicNumber: []byte{'Q', 'F', 'I', 0xfb},

diff --git a/pkg/image/validate.go b/pkg/image/validate.go
@@ -21,6 +21,8 @@ const (
 	ExtTar = ".tar"
 	// ExtXz is a constant for the .xz extenstion
 	ExtXz = ".xz"
+	// ExtZst is a constant for the .zst extenstion
+	ExtZst = ".zst"
 	// ExtTarXz is a constant for the .tar.xz extenstion
 	ExtTarXz = ExtTar + ExtXz
 	// ExtTarGz is a constant for the .tar.gz extenstion

diff --git a/pkg/importer/BUILD.bazel b/pkg/importer/BUILD.bazel
@@ -34,6 +34,7 @@ go_library(
         "//vendor/github.com/containers/image/v5/oci/archive:go_default_library",
         "//vendor/github.com/containers/image/v5/pkg/blobinfocache:go_default_library",
         "//vendor/github.com/containers/image/v5/types:go_default_library",
+        "//vendor/github.com/klauspost/compress/zstd:go_default_library",
         "//vendor/github.com/ovirt/go-ovirt:go_default_library",
         "//vendor/github.com/ovirt/go-ovirt-client:go_default_library",
         "//vendor/github.com/ovirt/go-ovirt-client-log-klog:go_default_library",

diff --git a/pkg/importer/format-readers.go b/pkg/importer/format-readers.go
@@ -23,6 +23,7 @@ import (
 	"io"
 	"strconv"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/ulikunitz/xz"
@@ -72,6 +73,7 @@ type FormatReaders struct {
 	Archived       bool
 	ArchiveXz      bool
 	ArchiveGz      bool
+	ArchiveZstd    bool
 	progressReader *prometheusutil.ProgressReader
 }
 
@@ -167,6 +169,12 @@ func (fr *FormatReaders) fileFormatSelector(hdr *image.Header) {
 			fr.Archived = true
 			fr.ArchiveGz = true
 		}
+	case "zst":
+		r, err = fr.zstReader()
+		if err == nil {
+			fr.Archived = true
+			fr.ArchiveZstd = true
+		}
 	case "qcow2":
 		r, err = fr.qcow2NopReader(hdr)
 		fr.Convert = true
@@ -208,6 +216,15 @@ func (fr *FormatReaders) gzReader() (io.ReadCloser, error) {
 	return gz, nil
 }
 
+// Return the zst reader.
+func (fr *FormatReaders) zstReader() (io.ReadCloser, error) {
+	zst, err := zstd.NewReader(fr.TopReader())
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create zst reader")
+	}
+	return zst.IOReadCloser(), nil
+}
+
 // Return the size of the endpoint "through the eye" of the previous reader. Note: there is no
 // qcow2 reader so nil is returned so that nothing is appended to the reader stack.
 // Note: size is stored at offset 24 in the qcow2 header.

diff --git a/tests/utils/fileConversion.go b/tests/utils/fileConversion.go
@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/pkg/errors"
 	"github.com/ulikunitz/xz"
 
@@ -18,6 +19,7 @@ import (
 var formatTable = map[string]func(string, string, string) (string, error){
 	image.ExtGz:    toGz,
 	image.ExtXz:    toXz,
+	image.ExtZst:   toZst,
 	image.ExtTar:   toTar,
 	image.ExtQcow2: convertUsingQemuImg,
 	image.ExtVmdk:  convertUsingQemuImg,
@@ -111,6 +113,29 @@ func toGz(src, tgtDir, ext string) (string, error) {
 	return tgtPath, nil
 }
 
+func toZst(src, tgtDir, ext string) (string, error) {
+	tgtFile, tgtPath, _ := createTargetFile(src, tgtDir, image.ExtGz)
+	defer tgtFile.Close()
+
+	w, err := zstd.NewWriter(tgtFile)
+	if err != nil {
+		return "", errors.Wrapf(err, "Error getting zst writer for file %s", tgtPath)
+	}
+	defer w.Close()
+
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return "", errors.Wrapf(err, "Error opening file %s", src)
+	}
+	defer srcFile.Close()
+
+	_, err = io.Copy(w, srcFile)
+	if err != nil {
+		return "", errors.Wrapf(err, "Error writing to file %s", tgtPath)
+	}
+	return tgtPath, nil
+}
+
 func toXz(src, tgtDir, ext string) (string, error) {
 	tgtFile, tgtPath, _ := createTargetFile(src, tgtDir, image.ExtXz)
 	defer tgtFile.Close()

diff --git a/tools/cdi-func-test-file-host-init/main.go b/tools/cdi-func-test-file-host-init/main.go
@@ -44,12 +44,14 @@ func main() {
 		[]string{""},
 		[]string{".gz"},
 		[]string{".xz"},
+		[]string{".zst"},
 		[]string{".qcow2"},
 		[]string{".vmdk"},
 		[]string{".vhd"},
 		[]string{".vhdx"},
 		[]string{".qcow2", ".gz"},
 		[]string{".qcow2", ".xz"},
+		[]string{".qcow2", ".zst"},
 	}
 
 	if err := utils.CreateCertForTestService(util.GetNamespace(), serviceName, configMapName, *certDir, certFile, keyFile); err != nil {