Generate Package Custom Resource when Package Bundle is Created (vmwa…

…re-tanzu#2199)

* generate Package CR when package bundle is generated

Signed-off-by: Harish Yayi <yharish991@gmail.com>

* Address golang-ci lint failed checks

Signed-off-by: F. Gold <fgold@vmware.com>

* Fix bug and pick up subversion from package values or flag

Signed-off-by: F. Gold <fgold@vmware.com>

Co-authored-by: Harish Yayi <yharish991@gmail.com>

Makefile

@@ -727,15 +727,15 @@ prep-package-tools:
 
 .PHONY: package-bundle
 package-bundle: tools prep-package-tools ## Build one specific tar bundle package, needs PACKAGE_NAME VERSION
-	cd hack/packages/package-tools && $(GO) run main.go package-bundle generate --thick $(PACKAGE_NAME) --repository=$(PACKAGE_REPOSITORY) --version=$(PACKAGE_VERSION) --sub-version=$(PACKAGE_SUB_VERSION)
+	cd hack/packages/package-tools && $(GO) run main.go package-bundle generate --thick $(PACKAGE_NAME) --repository=$(PACKAGE_REPOSITORY) --version=$(PACKAGE_VERSION) --sub-version=$(PACKAGE_SUB_VERSION) --registry=$(OCI_REGISTRY)
 
 .PHONY: package-bundle-thin
 package-bundle-thin: tools prep-package-tools ## Build one specific tar bundle package, needs PACKAGE_NAME VERSION
 	cd hack/packages/package-tools && $(GO) run main.go package-bundle generate $(PACKAGE_NAME) --repository=$(PACKAGE_REPOSITORY) --version=$(PACKAGE_VERSION) --sub-version=$(PACKAGE_SUB_VERSION)
 
 .PHONY: package-bundles
 package-bundles: tools prep-package-tools ## Build tar bundles for multiple packages
-	cd hack/packages/package-tools && $(GO) run main.go package-bundle generate --all --thick --repository=$(PACKAGE_REPOSITORY) --version=$(PACKAGE_VERSION) --sub-version=$(PACKAGE_SUB_VERSION)
+	cd hack/packages/package-tools && $(GO) run main.go package-bundle generate --all --thick --repository=$(PACKAGE_REPOSITORY) --version=$(PACKAGE_VERSION) --sub-version=$(PACKAGE_SUB_VERSION) --registry=$(OCI_REGISTRY)
 
 .PHONY: package-bundles-thin
 package-bundles-thin: tools prep-package-tools ## Build tar bundles for multiple packages

hack/packages/package-tools/cmd/helpers.go

@@ -0,0 +1,47 @@
+// Copyright 2022 VMware, Inc. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/vmware-tanzu/tanzu-framework/hack/packages/package-tools/constants"
+)
+
+func readPackageValues(projectRootDir string) (PackageValues, error) {
+	var packageValues PackageValues
+
+	packageValuesData, err := os.ReadFile(filepath.Join(projectRootDir, constants.PackageValuesFilePath))
+	if err != nil {
+		return PackageValues{}, fmt.Errorf("couldn't read file %s: %w", packageValuesFile, err)
+	}
+
+	if err := yaml.Unmarshal(packageValuesData, &packageValues); err != nil {
+		return PackageValues{}, fmt.Errorf("error while unmarshaling: %w", err)
+	}
+
+	return packageValues, nil
+}
+
+// getPackageFromPackageValues returns the package definition from the package-values.yaml file.
+func getPackageFromPackageValues(projectRootDir, packageName string) (Package, error) {
+	packageValues, err := readPackageValues(projectRootDir)
+	if err != nil {
+		return Package{}, err
+	}
+
+	for i := range packageValues.Repositories {
+		packages := packageValues.Repositories[i].Packages
+		for _, pkg := range packages {
+			if pkg.Name == packageName {
+				return pkg, nil
+			}
+		}
+	}
+	return Package{}, nil
+}

hack/packages/package-tools/cmd/package-bundle-generate.go

@@ -31,6 +31,7 @@ var packageBundleGenerateCmd = &cobra.Command{
 func init() {
 	packageBundleCmd.AddCommand(packageBundleGenerateCmd)
 	packageBundleGenerateCmd.Flags().StringVar(&packageRepository, "repository", "", "Package repository of the package bundle being created")
+	packageBundleGenerateCmd.Flags().StringVar(&registry, "registry", "", "OCI registry where the package bundle image needs to be stored")
 	packageBundleGenerateCmd.Flags().StringVar(&version, "version", "", "Package bundle version")
 	packageBundleGenerateCmd.Flags().StringVar(&subVersion, "sub-version", "", "Package bundle subversion")
 	packageBundleGenerateCmd.Flags().BoolVar(&all, "all", false, "Generate all package bundles in a repository")
@@ -68,6 +69,16 @@ func runPackageBundleGenerate(cmd *cobra.Command, args []string) error {
 		if err := generatePackageBundle(projectRootDir, toolsBinDir, packageName, packagePath); err != nil {
 			return fmt.Errorf("couldn't generate the package bundle: %w", err)
 		}
+		pkg, err := getPackageFromPackageValues(projectRootDir, packageName)
+		if err != nil {
+			return err
+		}
+
+		buildPkgDir := filepath.Join(projectRootDir, "build", "packages")
+		pkgValsDir := filepath.Join(projectRootDir, constants.PackageValuesFilePath)
+		if err := generatePackageCR(projectRootDir, toolsBinDir, registry, buildPkgDir, pkgValsDir, &pkg); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -180,28 +191,23 @@ func generatePackageBundle(projectRootDir, toolsBinDir, packageName, packagePath
 }
 
 func generatePackageBundles(projectRootDir, toolsBinDir string) error {
-	packageValuesData, err := os.ReadFile(filepath.Join(projectRootDir, constants.PackageValuesFilePath))
+	packageValues, err := readPackageValues(projectRootDir)
 	if err != nil {
 		return fmt.Errorf("couldn't read file %s: %w", packageValuesFile, err)
 	}
 
-	packageValues := PackageValues{}
-	if err := yaml.Unmarshal(packageValuesData, &packageValues); err != nil {
-		return fmt.Errorf("error while unmarshaling: %w", err)
-	}
-
 	repository, found := packageValues.Repositories[packageRepository]
 	if !found {
 		return fmt.Errorf("%s repository not found", packageRepository)
 	}
 
-	for i, pkg := range repository.Packages {
-		fmt.Printf("Generating %q package bundle...\n", pkg.Name)
+	for i := range repository.Packages {
+		fmt.Printf("Generating %q package bundle...\n", repository.Packages[i].Name)
 		imagePackageVersion := version
 		if subVersion != "" {
 			imagePackageVersion = version + "_" + subVersion
 		}
-		packagePath := filepath.Join(projectRootDir, "packages", packageRepository, pkg.Name)
+		packagePath := filepath.Join(projectRootDir, "packages", packageRepository, repository.Packages[i].Name)
 		if err := utils.RunMakeTarget(packagePath, "configure-package"); err != nil {
 			return err
 		}
@@ -212,9 +218,9 @@ func generatePackageBundles(projectRootDir, toolsBinDir string) error {
 		}
 
 		// push the imgpkg bundle to local registry
-		lockOutputFile := pkg.Name + "-" + imagePackageVersion + "-lock-output.yaml"
+		lockOutputFile := repository.Packages[i].Name + "-" + imagePackageVersion + "-lock-output.yaml"
 		imgpkgCmd := exec.Command(filepath.Join(toolsBinDir, "imgpkg"),
-			"push", "-b", constants.LocalRegistryURL+"/"+pkg.Name+":"+imagePackageVersion,
+			"push", "-b", constants.LocalRegistryURL+"/"+repository.Packages[i].Name+":"+imagePackageVersion,
 			"--file", filepath.Join(packagePath, "bundle"),
 			"--lock-output", lockOutputFile) // #nosec G204
 
@@ -236,7 +242,11 @@ func generatePackageBundles(projectRootDir, toolsBinDir string) error {
 		}
 
 		packageValues.Repositories[packageRepository].Packages[i].Version = getPackageVersion(version)
-		packageValues.Repositories[packageRepository].Packages[i].Sha256 = utils.AfterString(bundleLock.Bundle.Image, constants.LocalRegistryURL+"/"+pkg.Name+"@sha256:")
+		packageValues.Repositories[packageRepository].Packages[i].Sha256 = utils.AfterString(
+			bundleLock.Bundle.Image,
+			constants.LocalRegistryURL+"/"+repository.Packages[i].Name+"@sha256:",
+		)
+		packageValues.Repositories[packageRepository].Packages[i].PackageSubVersion = subVersion
 		yamlData, err := yaml.Marshal(&packageValues)
 		if err != nil {
 			return fmt.Errorf("error while marshaling: %w", err)
@@ -248,10 +258,16 @@ func generatePackageBundles(projectRootDir, toolsBinDir string) error {
 			return err
 		}
 
-		if err := generatePackageBundle(projectRootDir, toolsBinDir, pkg.Name, packagePath); err != nil {
+		if err := generatePackageBundle(projectRootDir, toolsBinDir, repository.Packages[i].Name, packagePath); err != nil {
 			return fmt.Errorf("couldn't generate package bundle: %w", err)
 		}
 
+		buildPkgsDir := filepath.Join(projectRootDir, "build", "packages")
+		pkgValsPath := filepath.Join(projectRootDir, constants.PackageValuesFilePath)
+		if err := generatePackageCR(projectRootDir, toolsBinDir, registry, buildPkgsDir, pkgValsPath, &repository.Packages[i]); err != nil {
+			return err
+		}
+
 		// remove lock output files
 		os.Remove(lockOutputFile)
 

hack/packages/package-tools/cmd/repo-bundle-generate.go

@@ -130,6 +130,7 @@ func generatePackageBundlesSha256(projectRootDir, localRegistry string) error {
 
 		packageValues.Repositories[packageRepository].Packages[i].Version = getPackageVersion(version)
 		packageValues.Repositories[packageRepository].Packages[i].Sha256 = utils.AfterString(bundleLock.Bundle.Image, localRegistry+"/"+pkg.Name+"@sha256:")
+		packageValues.Repositories[packageRepository].Packages[i].PackageSubVersion = subVersion
 		yamlData, err := yaml.Marshal(&packageValues)
 		if err != nil {
 			return fmt.Errorf("error while marshaling: %w", err)
@@ -201,8 +202,9 @@ func generateRepoBundle(projectRootDir string) error {
 		return fmt.Errorf("%s repository not found", packageRepository)
 	}
 
+	pkgRepoPkgsDir := filepath.Join(projectRootDir, constants.RepoBundlesDir, packageRepository, "packages")
 	for i := range repository.Packages {
-		if err := generatePackageCR(projectRootDir, toolsBinDir, &repository.Packages[i]); err != nil {
+		if err := generatePackageCR(projectRootDir, toolsBinDir, registry, pkgRepoPkgsDir, packageValuesFile, &repository.Packages[i]); err != nil {
 			return fmt.Errorf("couldn't generate the package: %w", err)
 		}
 	}
@@ -222,15 +224,23 @@ func generateRepoBundle(projectRootDir string) error {
 	return nil
 }
 
-func generatePackageCR(projectRootDir, toolsBinDir string, pkg *Package) error {
+func generatePackageCR(projectRootDir, toolsBinDir, registry, packageArtifactDirectory, packageValuesFile string, pkg *Package) error {
+	// package values file
 	fmt.Printf("Generating Package CR for package %q...\n", pkg.Name)
-	if err := utils.CreateDir(filepath.Join(projectRootDir, constants.RepoBundlesDir, packageRepository, "packages", pkg.Name+"."+pkg.Domain)); err != nil {
+	if err := utils.CreateDir(filepath.Join(packageArtifactDirectory, pkg.Name+"."+pkg.Domain)); err != nil {
 		return err
 	}
+
 	pkgVersion := getPackageVersion(version)
+	packageSubVersion := pkg.PackageSubVersion
+	if subVersion != "" {
+		// subVersion flag overrides package values subversion.
+		packageSubVersion = subVersion
+	}
+
 	packageFileName := pkgVersion + ".yml"
-	if pkg.PackageSubVersion != "" {
-		packageFileName = pkgVersion + pkg.PackageSubVersion + ".yml"
+	if packageSubVersion != "" {
+		packageFileName = pkgVersion + "+" + packageSubVersion + ".yml"
 	}
 
 	// generate Package CR and write it to a file
@@ -241,9 +251,12 @@ func generatePackageCR(projectRootDir, toolsBinDir string, pkg *Package) error {
 		"-v", "packageRepository="+packageRepository,
 		"-v", "packageName="+pkg.Name,
 		"-v", "registry="+registry,
-		"-v", "timestamp="+utils.GetFormattedCurrentTime()) // #nosec G204
+		"-v", "timestamp="+utils.GetFormattedCurrentTime(),
+		"-v", "version="+pkgVersion,
+		"-v", "subVersion="+packageSubVersion,
+	) // #nosec G204
 
-	packageFilePath := filepath.Join(projectRootDir, constants.RepoBundlesDir, packageRepository, "packages", pkg.Name+"."+pkg.Domain, packageFileName)
+	packageFilePath := filepath.Join(packageArtifactDirectory, pkg.Name+"."+pkg.Domain, packageFileName)
 	packageFile, err := os.Create(packageFilePath)
 	if err != nil {
 		return fmt.Errorf("couldn't create file %s: %w", packageFilePath, err)
@@ -267,7 +280,7 @@ func generatePackageCR(projectRootDir, toolsBinDir string, pkg *Package) error {
 		"-v", "packageName="+pkg.Name,
 		"-v", "registry="+registry) // #nosec G204
 
-	packageMetadataFilePath := filepath.Join(projectRootDir, constants.RepoBundlesDir, packageRepository, "packages", pkg.Name+"."+pkg.Domain, "metadata.yml")
+	packageMetadataFilePath := filepath.Join(packageArtifactDirectory, pkg.Name+"."+pkg.Domain, "metadata.yml")
 	metadataFile, err := os.Create(packageMetadataFilePath)
 	if err != nil {
 		return fmt.Errorf("couldn't create file %s: %w", packageMetadataFilePath, err)
@@ -279,7 +292,7 @@ func generatePackageCR(projectRootDir, toolsBinDir string, pkg *Package) error {
 
 	err = packageMetadataYttCmd.Run()
 	if err != nil {
-		return fmt.Errorf("couldn't generate PackageMetadata CR %s: %s", pkg.Name, packageYttCmdErrBytes.String())
+		return fmt.Errorf("couldn't generate PackageMetadata CR %s: %s", pkg.Name, packageMetadataYttCmdErrBytes.String())
 	}
 	return nil
 }

hack/packages/templates/repo-utils/images-tmpl.yaml

@@ -1,7 +1,7 @@
 #@ load("@ytt:data", "data")
 #@ load("package-helpers.lib.yaml", "get_package_repository")
 
-#@ package_repository = get_package_repository(data.values.packageRepository)
+#@ package_repository = get_package_repository(data.values.packageRepository, "")
 
 ---
 apiVersion: imgpkg.carvel.dev/v1alpha1

hack/packages/templates/repo-utils/package-cr-overlay.yaml

@@ -2,20 +2,44 @@
 #@ load("@ytt:overlay", "overlay")
 #@ load("package-helpers.lib.yaml", "get_package_repository", "get_package", "get_package_spec")
 
-#@ package_repository = get_package_repository(data.values.packageRepository)
+#@ package_repository = get_package_repository(data.values.packageRepository, data.values.packageName)
 #@ package = get_package(package_repository, data.values.packageName)
 #@ packageSpec = get_package_spec(package_repository, package)
 
-#@ if not hasattr(package, "packageSubVersion") or package.packageSubVersion == "":
-#@ packageVersion = package.version
+#@ if not hasattr(package, "packageSubVersion") and data.values.subVersion == "":
+#@  if package.version == "latest":
+#@    packageVersion = data.values.version
+#@  else:
+#@    packageVersion = package.version
+#@  end
 #@ else:
-#@ packageVersion = package.version + "+" + package.packageSubVersion
+#@  subVersion = data.values.subVersion
+#@  if subVersion == "":
+#@    subVersion = package.packageSubVersion
+#@  end
+#@  if package.version == "latest":
+#@    packageVersion = data.values.version + "+" + subVersion
+#@  else:
+#@    packageVersion = package.version + "+" + subVersion
+#@  end
 #@ end
 
-#@ if not hasattr(package, "packageSubVersion") or package.packageSubVersion == "":
-#@ imagePackageVersion = "v" + package.version
+#@ if not hasattr(package, "packageSubVersion") and data.values.subVersion == "":
+#@  if package.version == "latest":
+#@    imagePackageVersion = "v" + data.values.version
+#@  else:
+#@    imagePackageVersion = "v" + package.version
+#@  end
 #@ else:
-#@ imagePackageVersion = "v" + package.version + "_" + package.packageSubVersion
+#@  subVersion = data.values.subVersion
+#@  if subVersion == "":
+#@    subVersion = package.packageSubVersion
+#@  end
+#@  if package.version == "latest":
+#@    imagePackageVersion = "v" + data.values.version + "_" + subVersion
+#@  else:
+#@    imagePackageVersion = "v" + package.version + "_" + subVersion
+#@  end
 #@ end
 
 #@ packageLicense = "VMware’s End User License Agreement (Underlying OSS license: Apache License 2.0)"
@@ -35,7 +59,7 @@ spec:
   #@overlay/match missing_ok=True
   #@overlay/replace
   licenses:
-  - #@ packageLicense
+    -  #@ packageLicense
   template:
     spec:
       #@ if/end packageSpec:
@@ -58,11 +82,11 @@ spec:
             #@overlay/match missing_ok=True
             rawOptions:
               #@overlay/match by=lambda indexOrKey, left, right: "wait-timeout" in left, missing_ok=True
-              - #@ "--wait-timeout={}".format(packageSpec.deploy.kappWaitTimeout)
+              -  #@ "--wait-timeout={}".format(packageSpec.deploy.kappWaitTimeout)
               #@overlay/match by=lambda indexOrKey, left, right: "kube-api-qps" in left, missing_ok=True
-              - #@ "--kube-api-qps={}".format(packageSpec.deploy.kubeAPIQPS)
+              -  #@ "--kube-api-qps={}".format(packageSpec.deploy.kubeAPIQPS)
               #@overlay/match by=lambda indexOrKey, left, right: "kube-api-burst" in left, missing_ok=True
-              - #@ "--kube-api-burst={}".format(packageSpec.deploy.kubeAPIBurst)
+              -  #@ "--kube-api-burst={}".format(packageSpec.deploy.kubeAPIBurst)
             #@ end
   #@overlay/match missing_ok=True
   valuesSchema:

hack/packages/templates/repo-utils/package-helpers.lib.yaml

@@ -1,17 +1,27 @@
 #@ load("@ytt:data", "data")
 #@ load("@ytt:assert", "assert")
 
-#@ def get_package_repository(repository_name):
-#@   return data.values.repositories[repository_name]
+#@ def get_package_repository(repository_name, package_name):
+#@   if repository_name == "":
+#@    for repository in data.values.repositories:
+#@      for package in data.values.repositories[repository].packages:
+#@        if package.name == package_name:
+#@          return data.values.repositories[repository]
+#@        end
+#@      end
+#@    end
+#@   else:
+#@     return data.values.repositories[repository_name]
+#@   end
 #@ end
 
 #@ def get_package(package_repository, package_name):
-#@   for package in package_repository.packages:
-#@     if package.name == package_name:
-#@       return package
-#@     end
-#@   end
-#@   return None
+#@  for package in package_repository.packages:
+#@    if package.name == package_name:
+#@      return package
+#@    end
+#@  end
+#@  return None
 #@ end
 
 #@ def get_package_spec(package_repository, package):

hack/packages/templates/repo-utils/package-metadata-cr-overlay.yaml

@@ -2,7 +2,7 @@
 #@ load("@ytt:overlay", "overlay")
 #@ load("package-helpers.lib.yaml", "get_package_repository", "get_package")
 
-#@ package_repository = get_package_repository(data.values.packageRepository)
+#@ package_repository = get_package_repository(data.values.packageRepository, data.values.packageName)
 #@ package = get_package(package_repository, data.values.packageName)
 
 #@overlay/match by=overlay.subset({"kind":"PackageMetadata"}),expects=1

hack/packages/templates/repo-utils/packagerepo-tmpl.yaml

@@ -1,7 +1,7 @@
 #@ load("@ytt:data", "data")
 #@ load("package-helpers.lib.yaml", "get_package_repository")
 
-#@ package_repository = get_package_repository(data.values.packageRepository)
+#@ package_repository = get_package_repository(data.values.packageRepository, "")
 
 ---
 apiVersion: packaging.carvel.dev/v1alpha1