Merge pull request #651 from awels/calculate_available_space_before_i…

…mport-b

Check space before importing to report correct image resize size.

cmd/cdi-importer/importer.go

@@ -70,11 +70,12 @@ func main() {
 		source,
 		contentType,
 		imageSize,
+		util.GetAvailableSpace(common.ImporterVolumePath),
 	}
 
 	if source == controller.SourceNone && contentType == string(cdiv1.DataVolumeKubeVirt) {
 		requestImageSizeQuantity := resource.MustParse(imageSize)
-		minSizeQuantity := util.MinQuantity(resource.NewScaledQuantity(util.GetAvailableSpace(common.ImporterVolumePath), 0), &requestImageSizeQuantity)
+		minSizeQuantity := util.MinQuantity(resource.NewScaledQuantity(dso.AvailableSpace, 0), &requestImageSizeQuantity)
 		if minSizeQuantity.Cmp(requestImageSizeQuantity) != 0 {
 			// Available dest space is smaller than the size we want to create
 			glog.Warningf("Available space less than requested size, creating blank image sized to available space: %s.\n", minSizeQuantity.String())

pkg/importer/dataStream.go

@@ -94,6 +94,8 @@ type DataStreamOptions struct {
 	ContentType string
 	// ImageSize is the size we want the resulting image to be.
 	ImageSize string
+	// Available space is the available space before downloading the image
+	AvailableSpace int64
 }
 
 const (
@@ -134,6 +136,7 @@ func newDataStreamFromStream(stream io.ReadCloser) (*DataStream, error) {
 		controller.SourceHTTP,
 		string(cdiv1.DataVolumeKubeVirt),
 		"", // Blank means don't resize
+		util.GetAvailableSpace(common.ImporterVolumePath),
 	}, stream)
 }
 
@@ -329,15 +332,15 @@ func SaveStream(stream io.ReadCloser, dest string) (int64, error) {
 // ResizeImage resizes the images to match the requested size. Sometimes provisioners misbehave and the available space
 // is not the same as the requested space. For those situations we compare the available space to the requested space and
 // use the smallest of the two values.
-func ResizeImage(dest, imageSize string) error {
+func ResizeImage(dest, imageSize string, totalTargetSpace int64) error {
 	info, err := qemuOperations.Info(dest)
 	if err != nil {
 		return err
 	}
 	if imageSize != "" {
 		currentImageSizeQuantity := resource.NewScaledQuantity(info.VirtualSize, 0)
 		newImageSizeQuantity := resource.MustParse(imageSize)
-		minSizeQuantity := util.MinQuantity(resource.NewScaledQuantity(util.GetAvailableSpace(dest), 0), &newImageSizeQuantity)
+		minSizeQuantity := util.MinQuantity(resource.NewScaledQuantity(totalTargetSpace, 0), &newImageSizeQuantity)
 		if minSizeQuantity.Cmp(newImageSizeQuantity) != 0 {
 			// Available dest space is smaller than the size we want to resize to
 			glog.Warningf("Available space less than requested size, resizing image to available space %s.\n", minSizeQuantity.String())
@@ -709,18 +712,18 @@ func (d *DataStream) copy(dest string) error {
 			return errors.Wrap(err, "Streaming qcow2 to raw conversion failed")
 		}
 		if d.ImageSize != "" {
-			err = ResizeImage(dest, d.ImageSize)
-		}
-		if err != nil {
-			return errors.Wrap(err, "Resize of image failed")
+			err := ResizeImage(dest, d.ImageSize, d.AvailableSpace)
+			if err != nil {
+				return errors.Wrap(err, "Resize of image failed")
+			}
 		}
 		return nil
 	}
-	return copy(d.topReader(), dest, d.qemu, d.ImageSize, targetSize)
+	return copy(d.topReader(), dest, d.qemu, d.ImageSize, targetSize, d.AvailableSpace)
 }
 
 // Copy the file using its Reader (r) to the passed-in destination (`out`).
-func copy(r io.Reader, out string, qemu bool, imageSize string, targetSize int64) error {
+func copy(r io.Reader, out string, qemu bool, imageSize string, targetSize, availableSpace int64) error {
 	out = filepath.Clean(out)
 	glog.V(2).Infof("copying image file to %q", out)
 	dest := out
@@ -763,7 +766,7 @@ func copy(r io.Reader, out string, qemu bool, imageSize string, targetSize int64
 		dest = out
 	}
 	if imageSize != "" {
-		err = ResizeImage(dest, imageSize)
+		err = ResizeImage(dest, imageSize, availableSpace)
 	}
 	if err != nil {
 		return errors.Wrap(err, "Resize of image failed")

pkg/importer/dataStream_test.go

@@ -52,6 +52,7 @@ var imageDir, _ = filepath.Abs(TestImagesDir)
 var cirrosFileName = "cirros-qcow2.img"
 var tinyCoreFileName = "tinyCore.iso"
 var archiveFileName = "archive.tar"
+var cirrosRaw = "cirros.raw"
 var archiveFileNameWithoutExt = strings.TrimSuffix(archiveFileName, filepath.Ext(archiveFileName))
 var cirrosFilePath = filepath.Join(imageDir, cirrosFileName)
 var tinyCoreFilePath = filepath.Join(imageDir, tinyCoreFileName)
@@ -120,7 +121,9 @@ var _ = Describe("Data Stream", func() {
 			secretKey,
 			controller.SourceHTTP,
 			contentType,
-			""})
+			"",
+			int64(1234567890),
+		})
 		if ds != nil && len(ds.Readers) > 0 {
 			defer ds.Close()
 		}
@@ -155,7 +158,9 @@ var _ = Describe("Data Stream", func() {
 			"",
 			controller.SourceHTTP,
 			string(cdiv1.DataVolumeKubeVirt),
-			"1G"})
+			"1G",
+			int64(1234567890),
+		})
 		Expect(err).NotTo(HaveOccurred())
 		By("Closing data stream")
 		err = ds.Close()
@@ -174,7 +179,9 @@ var _ = Describe("Data Stream", func() {
 			"",
 			controller.SourceHTTP,
 			string(cdiv1.DataVolumeKubeVirt),
-			"20M"})
+			"20M",
+			int64(1234567890),
+		})
 		if ds != nil && len(ds.Readers) > 0 {
 			defer ds.Close()
 		}
@@ -206,7 +213,9 @@ var _ = Describe("Data Stream", func() {
 			"",
 			controller.SourceHTTP,
 			contentType,
-			"20M"})
+			"20M",
+			int64(1234567890),
+		})
 		defer func() {
 			tempTestServer.Close()
 		}()
@@ -278,7 +287,9 @@ var _ = Describe("Copy", func() {
 				"",
 				controller.SourceHTTP,
 				string(cdiv1.DataVolumeKubeVirt),
-				""})
+				"",
+				int64(1234567890),
+			})
 			if !wantErr {
 				Expect(err).NotTo(HaveOccurred())
 			} else {
@@ -323,7 +334,9 @@ var _ = Describe("Copy", func() {
 				"",
 				controller.SourceHTTP,
 				string(cdiv1.DataVolumeKubeVirt),
-				"1G"})
+				"1G",
+				int64(1234567890),
+			})
 			if wantErr {
 				Expect(err).To(HaveOccurred())
 			} else {
@@ -352,7 +365,7 @@ var _ = Describe("Copy", func() {
 		defer os.Remove(out)
 		By("Replacing QEMU Operations")
 		replaceQEMUOperations(qemuOperations, func() {
-			err := copy(r, out, qemu, "", int64(500000000))
+			err := copy(r, out, qemu, "", int64(500000000), int64(50000000000))
 			if !wantErr {
 				Expect(err).NotTo(HaveOccurred())
 			} else {
@@ -400,6 +413,67 @@ var _ = Describe("http", func() {
 	})
 })
 
+var _ = Describe("ResizeImage", func() {
+	var tmpDir string
+	var err error
+
+	BeforeEach(func() {
+		tmpDir, err = ioutil.TempDir("", "imagedir")
+		Expect(err).NotTo(HaveOccurred())
+		input, err := ioutil.ReadFile(filepath.Join(imageDir, cirrosRaw))
+		Expect(err).NotTo(HaveOccurred())
+		err = ioutil.WriteFile(filepath.Join(tmpDir, cirrosRaw), input, 0644)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		os.Remove(tmpDir)
+	})
+
+	It("Should successfully resize, to smaller available", func() {
+		dest := filepath.Join(tmpDir, cirrosRaw)
+		err := ResizeImage(dest, "20M", int64(18874368))
+		Expect(err).NotTo(HaveOccurred())
+		info, err := qemuOperations.Info(dest)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(info.VirtualSize).To(Equal(int64(18874368)))
+	})
+
+	It("Should fail with invalid file", func() {
+		err := ResizeImage(filepath.Join(tmpDir, "invalid"), "20M", int64(18874368))
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("Should successfully resize even if more space available", func() {
+		dest := filepath.Join(tmpDir, cirrosRaw)
+		err := ResizeImage(dest, "20M", int64(200000000))
+		Expect(err).NotTo(HaveOccurred())
+		info, err := qemuOperations.Info(dest)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(info.VirtualSize).To(Equal(int64(20971520)))
+	})
+
+	It("Should successfully not resize if sizes are same.", func() {
+		dest := filepath.Join(tmpDir, cirrosRaw)
+		info, err := qemuOperations.Info(dest)
+		originalSize := info.VirtualSize
+		Expect(err).NotTo(HaveOccurred())
+		err = ResizeImage(dest, strconv.FormatInt(originalSize, 10), int64(200000000))
+		Expect(err).NotTo(HaveOccurred())
+		info, err = qemuOperations.Info(dest)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(info.VirtualSize).To(Equal(originalSize))
+	})
+
+	It("Should fail with valid file, but empty imageSize", func() {
+		dest := filepath.Join(tmpDir, cirrosRaw)
+		err := ResizeImage(dest, "", int64(200000000))
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(Equal("Image resize called with blank resize"))
+	})
+
+})
+
 var _ = Describe("close readers", func() {
 	type args struct {
 		readers []reader

pkg/importer/registry_test.go

@@ -37,7 +37,9 @@ var _ = Describe("Copy from Registry", func() {
 				"",
 				controller.SourceRegistry,
 				string(cdiv1.DataVolumeKubeVirt),
-				"1G"})
+				"1G",
+				int64(1234567890),
+			})
 			if !wantErr {
 				Expect(err).NotTo(HaveOccurred())
 			} else {