add http zstd reader for cdi (#2593)

* add http zstd reader for cdi Signed-off-by: huangzy <huangzynn@outlook.com> * add dv-https-import-zst test Signed-off-by: huangzy <huangzynn@outlook.com> * fix cdi-file-host tozst ext Signed-off-by: huangzy <huangzynn@outlook.com> * fix the URL to point to the right name tinyCore.iso.zst Signed-off-by: huangzy <huangzynn@outlook.com> --------- Signed-off-by: huangzy <huangzynn@outlook.com>
kubevirt · Mar 9, 2023 · 15e645a · 15e645a
1 parent 7788c9c
commit 15e645a
Show file tree

Hide file tree

Showing 9 changed files with 83 additions and 0 deletions.
diff --git a/pkg/image/filefmt.go b/pkg/image/filefmt.go
@@ -25,6 +25,12 @@ var knownHeaders = Headers{
 		SizeOff: 0,
 		SizeLen: 0,
 	},
+	"zst": Header{
+		Format:      "zst",
+		magicNumber: []byte{0x28, 0xb5, 0x2f, 0xfd},
+		SizeOff:     0,
+		SizeLen:     0,
+	},
 	"qcow2": Header{
 		Format:      "qcow2",
 		magicNumber: []byte{'Q', 'F', 'I', 0xfb},

diff --git a/pkg/image/validate.go b/pkg/image/validate.go
@@ -21,6 +21,8 @@ const (
 	ExtTar = ".tar"
 	// ExtXz is a constant for the .xz extenstion
 	ExtXz = ".xz"
+	// ExtZst is a constant for the .zst extenstion
+	ExtZst = ".zst"
 	// ExtTarXz is a constant for the .tar.xz extenstion
 	ExtTarXz = ExtTar + ExtXz
 	// ExtTarGz is a constant for the .tar.gz extenstion

diff --git a/pkg/importer/BUILD.bazel b/pkg/importer/BUILD.bazel
@@ -34,6 +34,7 @@ go_library(
         "//vendor/github.com/containers/image/v5/oci/archive:go_default_library",
         "//vendor/github.com/containers/image/v5/pkg/blobinfocache:go_default_library",
         "//vendor/github.com/containers/image/v5/types:go_default_library",
+        "//vendor/github.com/klauspost/compress/zstd:go_default_library",
         "//vendor/github.com/ovirt/go-ovirt:go_default_library",
         "//vendor/github.com/ovirt/go-ovirt-client:go_default_library",
         "//vendor/github.com/ovirt/go-ovirt-client-log-klog:go_default_library",

diff --git a/pkg/importer/format-readers.go b/pkg/importer/format-readers.go
@@ -23,6 +23,7 @@ import (
 	"io"
 	"strconv"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/ulikunitz/xz"
@@ -72,6 +73,7 @@ type FormatReaders struct {
 	Archived       bool
 	ArchiveXz      bool
 	ArchiveGz      bool
+	ArchiveZstd    bool
 	progressReader *prometheusutil.ProgressReader
 }
 
@@ -167,6 +169,12 @@ func (fr *FormatReaders) fileFormatSelector(hdr *image.Header) {
 			fr.Archived = true
 			fr.ArchiveGz = true
 		}
+	case "zst":
+		r, err = fr.zstReader()
+		if err == nil {
+			fr.Archived = true
+			fr.ArchiveZstd = true
+		}
 	case "qcow2":
 		r, err = fr.qcow2NopReader(hdr)
 		fr.Convert = true
@@ -208,6 +216,15 @@ func (fr *FormatReaders) gzReader() (io.ReadCloser, error) {
 	return gz, nil
 }
 
+// Return the zst reader.
+func (fr *FormatReaders) zstReader() (io.ReadCloser, error) {
+	zst, err := zstd.NewReader(fr.TopReader())
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create zst reader")
+	}
+	return zst.IOReadCloser(), nil
+}
+
 // Return the size of the endpoint "through the eye" of the previous reader. Note: there is no
 // qcow2 reader so nil is returned so that nothing is appended to the reader stack.
 // Note: size is stored at offset 24 in the qcow2 header.

diff --git a/tests/datavolume_test.go b/tests/datavolume_test.go
@@ -72,6 +72,9 @@ var _ = Describe("[vendor:cnv-qe@redhat.com][level:component]DataVolume tests",
 	httpsTinyCoreVhdxURL := func() string {
 		return fmt.Sprintf(utils.HTTPSTinyCoreVhdxURL, f.CdiInstallNs)
 	}
+	httpsTinyCoreZstURL := func() string {
+		return fmt.Sprintf(utils.HTTPSTinyCoreZstURL, f.CdiInstallNs)
+	}
 	tinyCoreQcow2URL := func() string {
 		return fmt.Sprintf(utils.TinyCoreQcow2URL+".gz", f.CdiInstallNs)
 	}
@@ -599,6 +602,30 @@ var _ = Describe("[vendor:cnv-qe@redhat.com][level:component]DataVolume tests",
 					Message: "Import Complete",
 					Reason:  "Completed",
 				}}),
+			table.Entry("[rfe_id:1115][crit:high][test_id:1379]succeed creating import dv with given valid zst url (https)", dataVolumeTestArguments{
+				name:             "dv-https-import-zst",
+				size:             "1Gi",
+				url:              httpsTinyCoreZstURL,
+				dvFunc:           createHTTPSDataVolume,
+				eventReason:      dvc.ImportSucceeded,
+				phase:            cdiv1.Succeeded,
+				checkPermissions: true,
+				readyCondition: &cdiv1.DataVolumeCondition{
+					Type:   cdiv1.DataVolumeReady,
+					Status: v1.ConditionTrue,
+				},
+				boundCondition: &cdiv1.DataVolumeCondition{
+					Type:    cdiv1.DataVolumeBound,
+					Status:  v1.ConditionTrue,
+					Message: "PVC dv-https-import-zst Bound",
+					Reason:  "Bound",
+				},
+				runningCondition: &cdiv1.DataVolumeCondition{
+					Type:    cdiv1.DataVolumeRunning,
+					Status:  v1.ConditionFalse,
+					Message: "Import Complete",
+					Reason:  "Completed",
+				}}),
 			table.Entry("succeed creating import dv with custom https cert that has a weird filename", dataVolumeTestArguments{
 				name:             "dv-https-import-qcow2",
 				size:             "1Gi",

diff --git a/tests/utils/BUILD.bazel b/tests/utils/BUILD.bazel
@@ -30,6 +30,7 @@ go_library(
         "//pkg/util/naming:go_default_library",
         "//staging/src/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library",
         "//staging/src/kubevirt.io/containerized-data-importer-api/pkg/apis/upload/v1beta1:go_default_library",
+        "//vendor/github.com/klauspost/compress/zstd:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",

diff --git a/tests/utils/datavolume.go b/tests/utils/datavolume.go
@@ -47,6 +47,8 @@ const (
 	HTTPSTinyCoreIsoURL = "https://cdi-file-host.%s/tinyCore.iso"
 	// HTTPSTinyCoreQcow2URL provides a test (https) url for the tineyCore qcow2 image
 	HTTPSTinyCoreQcow2URL = "https://cdi-file-host.%s/tinyCore.qcow2"
+	// HTTPSTinyCoreZstURL provides a test (https) url for the tineyCore zst image
+	HTTPSTinyCoreZstURL = "https://cdi-file-host.%s/tinyCore.iso.zst"
 	// TinyCoreQcow2URLRateLimit provides a test url for the tineyCore qcow2 image via rate-limiting proxy
 	TinyCoreQcow2URLRateLimit = "http://cdi-file-host.%s:82/tinyCore.qcow2"
 	// TinyCoreQcow2GzURLRateLimit provides a test url for the tineyCore qcow2.gz image via rate-limiting proxy

diff --git a/tests/utils/fileConversion.go b/tests/utils/fileConversion.go
@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/pkg/errors"
 	"github.com/ulikunitz/xz"
 
@@ -18,6 +19,7 @@ import (
 var formatTable = map[string]func(string, string, string) (string, error){
 	image.ExtGz:    toGz,
 	image.ExtXz:    toXz,
+	image.ExtZst:   toZst,
 	image.ExtTar:   toTar,
 	image.ExtQcow2: convertUsingQemuImg,
 	image.ExtVmdk:  convertUsingQemuImg,
@@ -111,6 +113,29 @@ func toGz(src, tgtDir, ext string) (string, error) {
 	return tgtPath, nil
 }
 
+func toZst(src, tgtDir, ext string) (string, error) {
+	tgtFile, tgtPath, _ := createTargetFile(src, tgtDir, image.ExtZst)
+	defer tgtFile.Close()
+
+	w, err := zstd.NewWriter(tgtFile)
+	if err != nil {
+		return "", errors.Wrapf(err, "Error getting zst writer for file %s", tgtPath)
+	}
+	defer w.Close()
+
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return "", errors.Wrapf(err, "Error opening file %s", src)
+	}
+	defer srcFile.Close()
+
+	_, err = io.Copy(w, srcFile)
+	if err != nil {
+		return "", errors.Wrapf(err, "Error writing to file %s", tgtPath)
+	}
+	return tgtPath, nil
+}
+
 func toXz(src, tgtDir, ext string) (string, error) {
 	tgtFile, tgtPath, _ := createTargetFile(src, tgtDir, image.ExtXz)
 	defer tgtFile.Close()

diff --git a/tools/cdi-func-test-file-host-init/main.go b/tools/cdi-func-test-file-host-init/main.go
@@ -44,12 +44,14 @@ func main() {
 		[]string{""},
 		[]string{".gz"},
 		[]string{".xz"},
+		[]string{".zst"},
 		[]string{".qcow2"},
 		[]string{".vmdk"},
 		[]string{".vhd"},
 		[]string{".vhdx"},
 		[]string{".qcow2", ".gz"},
 		[]string{".qcow2", ".xz"},
+		[]string{".qcow2", ".zst"},
 	}
 
 	if err := utils.CreateCertForTestService(util.GetNamespace(), serviceName, configMapName, *certDir, certFile, keyFile); err != nil {