Stream sha256

atrium/vestibulum/trcdb/trcplgtoolbase/trcplgtoolbase.go

@@ -10,10 +10,8 @@ import (
 	"log"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trimble-oss/tierceron/buildopts"
 	"github.com/trimble-oss/tierceron/buildopts/coreopts"
@@ -414,6 +412,7 @@ func CommonMain(envDefaultPtr *string,
 	pluginToolConfig["expandTargetPtr"] = *expandTargetPtr //is a bool that gets converted to a string for writeout/certify
 	pluginToolConfig["newrelicAppName"] = *newrelicAppNamePtr
 	pluginToolConfig["newrelicLicenseKey"] = *newrelicLicenseKeyPtr
+	pluginToolConfig["codebundledeployPtr"] = *codebundledeployPtr //for optimization change in certify
 
 	if _, ok := pluginToolConfig["trcplugin"].(string); !ok {
 		pluginToolConfig["trcplugin"] = pluginToolConfig["pluginNamePtr"].(string)
@@ -544,98 +543,6 @@ func CommonMain(envDefaultPtr *string,
 				return err
 			}
 		}
-
-		if pluginToolConfig["trcsha256"] != nil &&
-			pluginToolConfig["imagesha256"] != nil &&
-			pluginToolConfig["trcsha256"].(string) == pluginToolConfig["imagesha256"].(string) {
-			// Write the image to the destination...
-			var deployPath string
-			var deployRoot string
-			if deploySubPath, ok := pluginToolConfig["trcdeploysubpath"]; ok {
-				deployRoot = filepath.Join(pluginToolConfig["trcdeployroot"].(string), deploySubPath.(string))
-			} else {
-				deployRoot = pluginToolConfig["trcdeployroot"].(string)
-			}
-
-			//check if there is a place holder, if there is replace it
-			if strings.Contains(deployRoot, "{{.trcpathparam}}") {
-				if pathParam, ok := pluginToolConfig["trcpathparam"].(string); ok && pathParam != "" {
-					r, _ := regexp.Compile("^[a-zA-Z0-9_]*$")
-					if !r.MatchString(pathParam) {
-						fmt.Println("trcpathparam can only contain alphanumberic characters or underscores")
-						return errors.New("trcpathparam can only contain alphanumberic characters or underscores")
-					}
-					deployRoot = strings.Replace(deployRoot, "{{.trcpathparam}}", pathParam, -1)
-				} else {
-					return errors.New("Unable to replace path placeholder with pathParam.")
-				}
-			}
-			deployPath = filepath.Join(deployRoot, pluginToolConfig["trccodebundle"].(string))
-
-			fmt.Printf("Deploying image to: %s\n", deployPath)
-
-			if _, err = os.Stat(deployRoot); err != nil {
-				err = os.MkdirAll(deployRoot, 0644)
-				if err != nil {
-					fmt.Println(err.Error())
-					fmt.Println("Could not prepare needed directory for deployment.")
-					return err
-				}
-			}
-
-			err = os.WriteFile(deployPath, pluginToolConfig["rawImageFile"].([]byte), 0644)
-			if err != nil {
-				fmt.Println(err.Error())
-				fmt.Println("Image write failure.")
-				return err
-			}
-
-			if expandTarget, ok := pluginToolConfig["trcexpandtarget"].(string); ok && expandTarget == "true" {
-				// TODO: provide archival of existing directory.
-				if ok, errList := trcvutils.UncompressZipFile(deployPath); !ok {
-					fmt.Printf("Uncompressing zip file in place failed. %v\n", errList)
-					return errList[0]
-				} else {
-					os.Remove(deployPath)
-				}
-			} else {
-				if strings.HasSuffix(deployPath, ".war") {
-					explodedWarPath := strings.TrimSuffix(deployPath, ".war")
-					fmt.Printf("Checking exploded war path: %s\n", explodedWarPath)
-					if _, err := os.Stat(explodedWarPath); err == nil {
-						if deploySubPath, ok := pluginToolConfig["trcdeploysubpath"]; ok {
-							archiveDirPath := filepath.Join(deployRoot, "archive")
-							fmt.Printf("Verifying archive directory: %s\n", archiveDirPath)
-							err := os.MkdirAll(archiveDirPath, 0700)
-							if err == nil {
-								currentTime := time.Now()
-								formattedTime := fmt.Sprintf("%d-%02d-%02d_%02d-%02d-%02d", currentTime.Year(), currentTime.Month(), currentTime.Day(), currentTime.Hour(), currentTime.Minute(), currentTime.Second())
-								archiveRoot := filepath.Join(pluginToolConfig["trcdeployroot"].(string), deploySubPath.(string), "archive", formattedTime)
-								fmt.Printf("Verifying archive backup directory: %s\n", archiveRoot)
-								err := os.MkdirAll(archiveRoot, 0700)
-								if err == nil {
-									archivePath := filepath.Join(archiveRoot, pluginToolConfig["trccodebundle"].(string))
-									archivePath = strings.TrimSuffix(archivePath, ".war")
-									fmt.Printf("Archiving: %s to %s\n", explodedWarPath, archivePath)
-									os.Rename(explodedWarPath, archivePath)
-								}
-							}
-						}
-					}
-				}
-			}
-
-			fmt.Printf("Image deployed to: %s\n", deployPath)
-		} else {
-			errMessage := fmt.Sprintf("image not certified.  cannot deploy image for %s", pluginToolConfig["trcplugin"])
-			if trcshDriverConfigBase.FeatherCtx != nil {
-				fmt.Printf("%s\n", errMessage)
-				trcshDriverConfigBase.FeatherCtx.Log.Printf(errMessage)
-			} else {
-				fmt.Printf("%s\n", errMessage)
-			}
-			return errors.New(errMessage)
-		}
 	} else if *certifyImagePtr {
 		//Certify Image
 		carrierCertify := false

pkg/core/util/repository/azrcr.go

@@ -4,9 +4,12 @@
 package repository
 
 import (
+	"archive/tar"
 	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -110,48 +113,68 @@ foundTag:
 		}
 	}
 
+	if pluginToolConfig["imagesha256"] == nil || pluginToolConfig["trcsha256"] == nil || pluginToolConfig["imagesha256"].(string) != pluginToolConfig["trcsha256"].(string) {
+		if pluginToolConfig["codebundledeployPtr"] != nil && pluginToolConfig["codebundledeployPtr"].(bool) {
+			errMessage := fmt.Sprintf("image not certified.  cannot deploy image for %s", pluginToolConfig["trcplugin"])
+			return errors.New(errMessage)
+		}
+	}
 	return nil
 }
 
 func GetImageShaFromLayer(blobClient *azcontainerregistry.BlobClient, name string, digest string, pluginToolConfig map[string]interface{}) (string, error) {
-
 	configRes, err := blobClient.GetBlob(context.Background(), name, digest, nil)
 	if err != nil {
 		return "", errors.New("Failed to get config:" + err.Error())
 	}
-
+	writingFile := false
+	sha := ""
+writeToFile:
 	reader, readErr := azcontainerregistry.NewDigestValidationReader(digest, configRes.BlobData)
 	if readErr != nil {
 		return "", errors.New("Failed to create validation reader" + readErr.Error())
 	}
 
-	layerData, configErr := io.ReadAll(reader)
-	if configErr != nil {
-		return "", errors.New("Failed to read config data:" + configErr.Error())
-	}
-
-	pluginTarredData, gUnZipError := gUnZipData(&layerData)
+	gzipReader, gUnZipError := gzip.NewReader(reader)
 	if gUnZipError != nil {
 		return "", errors.New("gunzip failed")
 	}
-	pluginImage, gUnTarError := untarData(&pluginTarredData)
-	if gUnTarError != nil {
-		return "", errors.New("untarring failed")
-	}
-	pluginSha := sha256.Sum256(pluginImage)
-	sha256 := fmt.Sprintf("%x", pluginSha)
-	if pluginToolConfig != nil {
-		if _, ok := pluginToolConfig["trcsha256"]; !ok {
-			// Not looking for anything in particular so just grab the last image.
-			pluginToolConfig["rawImageFile"] = pluginImage
+
+	tarReader := tar.NewReader(gzipReader)
+	if writingFile {
+		writingFile = false
+		err := deployImage(tarReader, pluginToolConfig)
+		if err != nil {
+			fmt.Println("Unable to deploy image.")
+			return sha, err
 		} else {
-			if pluginToolConfig["trcsha256"].(string) == sha256 {
-				pluginToolConfig["rawImageFile"] = pluginImage
+			return sha, nil
+		}
+	} else {
+		hash := sha256.New()
+		for {
+			_, err := tarReader.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return "", err
+			}
+			if _, err := io.Copy(hash, tarReader); err != nil {
+				return "", err
 			}
 		}
+		sha = hex.EncodeToString(hash.Sum(nil))
+	}
+
+	if pluginToolConfig["trcsha256"] != nil && pluginToolConfig["trcsha256"].(string) == sha {
+		if pluginToolConfig["codebundledeployPtr"] != nil && pluginToolConfig["codebundledeployPtr"].(bool) {
+			writingFile = true
+			goto writeToFile
+		}
 	}
 
-	return sha256, nil
+	return sha, nil
 }
 
 // Return url to the image to be used for download.

pkg/core/util/repository/ecrcommon.go

@@ -4,10 +4,34 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
+	"errors"
+	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	trcvutils "github.com/trimble-oss/tierceron/pkg/core/util"
 )
 
+type StreamingTarReader struct {
+	tarReader *tar.Reader
+}
+
+func NewStreamingTarReader(tr *tar.Reader) *StreamingTarReader {
+	return &StreamingTarReader{tr}
+}
+
+// change type of data for gzip reader
+func (str *StreamingTarReader) read(data []byte) error {
+	for {
+		// read until eof --> then call tr.Next() and repeat
+	}
+}
+
 func gUnZipData(data *[]byte) ([]byte, error) {
 	var unCompressedBytes []byte
 	newB := bytes.NewBuffer(unCompressedBytes)
@@ -61,3 +85,110 @@ func getImage(downloadUrl string) (*[]byte, error) {
 
 	return &body, nil
 }
+
+func deployImage(reader *tar.Reader, pluginToolConfig map[string]interface{}) error {
+	// Write the image to the destination...
+	var deployPath string
+	var deployRoot string
+	if deploySubPath, ok := pluginToolConfig["trcdeploysubpath"]; ok {
+		deployRoot = filepath.Join(pluginToolConfig["trcdeployroot"].(string), deploySubPath.(string))
+	} else {
+		deployRoot = pluginToolConfig["trcdeployroot"].(string)
+	}
+
+	//check if there is a place holder, if there is replace it
+	if strings.Contains(deployRoot, "{{.trcpathparam}}") {
+		if pathParam, ok := pluginToolConfig["trcpathparam"].(string); ok && pathParam != "" {
+			r, _ := regexp.Compile("^[a-zA-Z0-9_]*$")
+			if !r.MatchString(pathParam) {
+				fmt.Println("trcpathparam can only contain alphanumberic characters or underscores")
+				return errors.New("trcpathparam can only contain alphanumberic characters or underscores")
+			}
+			deployRoot = strings.Replace(deployRoot, "{{.trcpathparam}}", pathParam, -1)
+		} else {
+			return errors.New("Unable to replace path placeholder with pathParam.")
+		}
+	}
+	deployPath = filepath.Join(deployRoot, pluginToolConfig["trccodebundle"].(string))
+
+	fmt.Printf("Deploying image to: %s\n", deployPath)
+
+	if _, err := os.Stat(deployRoot); err != nil {
+		err = os.MkdirAll(deployRoot, 0644)
+		if err != nil {
+			fmt.Println(err.Error())
+			fmt.Println("Could not prepare needed directory for deployment.")
+			return err
+		}
+	}
+
+	f, err := os.OpenFile(deployPath, os.O_RDWR|os.O_CREATE, 0644)
+	if err != nil {
+		fmt.Println("Could not create file for deployment.")
+		return err
+	}
+
+	defer func(f *os.File) {
+		close_err := f.Close()
+		if err == nil && close_err != nil {
+			fmt.Println("Error closing file for deployment")
+			err = close_err
+		}
+	}(f)
+
+	for {
+		_, err := reader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			fmt.Println(err.Error())
+			fmt.Println("Image write failure.")
+			return err
+		}
+
+		_, err = io.Copy(f, reader)
+		if err != nil {
+			fmt.Println(err.Error())
+			fmt.Println("Image write failure.")
+			return err
+		}
+	}
+
+	if expandTarget, ok := pluginToolConfig["trcexpandtarget"].(string); ok && expandTarget == "true" {
+		// TODO: provide archival of existing directory.
+		if ok, errList := trcvutils.UncompressZipFile(deployPath); !ok {
+			fmt.Printf("Uncompressing zip file in place failed. %v\n", errList)
+			return errList[0]
+		} else {
+			os.Remove(deployPath)
+		}
+	} else {
+		if strings.HasSuffix(deployPath, ".war") {
+			explodedWarPath := strings.TrimSuffix(deployPath, ".war")
+			fmt.Printf("Checking exploded war path: %s\n", explodedWarPath)
+			if _, err := os.Stat(explodedWarPath); err == nil {
+				if deploySubPath, ok := pluginToolConfig["trcdeploysubpath"]; ok {
+					archiveDirPath := filepath.Join(deployRoot, "archive")
+					fmt.Printf("Verifying archive directory: %s\n", archiveDirPath)
+					err := os.MkdirAll(archiveDirPath, 0700)
+					if err == nil {
+						currentTime := time.Now()
+						formattedTime := fmt.Sprintf("%d-%02d-%02d_%02d-%02d-%02d", currentTime.Year(), currentTime.Month(), currentTime.Day(), currentTime.Hour(), currentTime.Minute(), currentTime.Second())
+						archiveRoot := filepath.Join(pluginToolConfig["trcdeployroot"].(string), deploySubPath.(string), "archive", formattedTime)
+						fmt.Printf("Verifying archive backup directory: %s\n", archiveRoot)
+						err := os.MkdirAll(archiveRoot, 0700)
+						if err == nil {
+							archivePath := filepath.Join(archiveRoot, pluginToolConfig["trccodebundle"].(string))
+							archivePath = strings.TrimSuffix(archivePath, ".war")
+							fmt.Printf("Archiving: %s to %s\n", explodedWarPath, archivePath)
+							os.Rename(explodedWarPath, archivePath)
+						}
+					}
+				}
+			}
+		}
+	}
+	fmt.Printf("Image deployed to: %s\n", deployPath)
+	return nil
+}