Skip to content

Commit

Permalink
Implement gzip for dag bundle compression for cloud deploys (#1778)
Browse files Browse the repository at this point in the history
Co-authored-by: Pritesh Arora <pritt@Priteshs-MacBook-Pro.local>
  • Loading branch information
2 people authored and neel-astro committed Jan 8, 2025
1 parent 97d5768 commit 1c9ff3e
Showing 1 changed file with 20 additions and 7 deletions.
27 changes: 20 additions & 7 deletions cloud/deploy/bundle.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,18 @@ func DeleteBundle(input *DeleteBundleInput) error {

func uploadBundle(tarDirPath, bundlePath, uploadURL string, prependBaseDir bool) (string, error) {
tarFilePath := filepath.Join(tarDirPath, "bundle.tar")
tarGzFilePath := tarFilePath + ".gz"
defer func() {
err := os.Remove(tarFilePath)
if err != nil {
fmt.Println("\nFailed to delete tar file: ", err.Error())
fmt.Println("\nPlease delete the tar file manually from path: " + tarFilePath)
tarFiles := []string{tarFilePath, tarGzFilePath}
for _, file := range tarFiles {
err := os.Remove(file)
if err != nil {
if os.IsNotExist(err) {
continue
}
fmt.Println("\nFailed to delete archived file: ", err.Error())
fmt.Println("\nPlease delete the archived file manually from path: " + file)
}
}
}()

Expand All @@ -156,13 +163,19 @@ func uploadBundle(tarDirPath, bundlePath, uploadURL string, prependBaseDir bool)
return "", err
}

tarFile, err := os.Open(tarFilePath)
// Gzip the tar
err = fileutil.GzipFile(tarFilePath, tarGzFilePath)
if err != nil {
return "", err
}

tarGzFile, err := os.Open(tarGzFilePath)
if err != nil {
return "", err
}
defer tarFile.Close()
defer tarGzFile.Close()

versionID, err := azureUploader(uploadURL, tarFile)
versionID, err := azureUploader(uploadURL, tarGzFile)
if err != nil {
return "", err
}
Expand Down

0 comments on commit 1c9ff3e

Please sign in to comment.