Add support for building composite buildpacks

This PR resolves #48.

In addition, it fixes a bug where a missing `BP_ROOT` was ignored, which was confusing. It also does a little refactoring renaming some variables to better indicate intent.

Signed-off-by: Daniel Mikusa <dan@mikusa.com>

Makefile

@@ -3,7 +3,7 @@ GOCMD?=go
 GO_VERSION=$(shell go list -m -f "{{.GoVersion}}")
 LIBPAKTOOLS_VERSION=$(shell ./scripts/version.sh)
 PACKAGE_BASE=github.com/paketo-buildpacks/libpak-tools
-OUTDIR=./binaries
+OUTDIR=$(HOME)/go/bin
 LDFLAGS="-s -w"
 
 all: test libpak-tools
@@ -15,13 +15,6 @@ libpak-tools: out
 	@echo "> Building libpak-tools..."
 	go build -ldflags=$(LDFLAGS) -o $(OUTDIR)/libpak-tools main.go
 
-package: OUTDIR=./bin
-package: clean libpak-tools
-	@echo "> Packaging up binaries..."
-	mkdir -p dist/
-	tar czf dist/libpak-tools-$(LIBPAKTOOLS_VERSION).tgz $(OUTDIR)/*
-	rm -rf ./bin
-
 install-goimports:
 	@echo "> Installing goimports..."
 	cd tools && $(GOCMD) install golang.org/x/tools/cmd/goimports

README.md

@@ -2,6 +2,15 @@
 
 This repository pulls together and publishes a number of helpful tools for the management and release of buildpacks.
 
+## Configuration Environment Variables
+
+| Name                  | Default                                    | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| --------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `BP_ROOT`             | ``                                         | The location where you have `git clone`'d all of the buildpacks. The structure should be `$BP_ROOT/<github_org>/<github_repo>`. For example: `$BP_ROOT/paketo-buildpacks/bellsoft-liberica`. This setting is required *if* you want the tool to infer where your buildpacks live based on the `--buildpack-id` you supply. If you do not include it, then you need to include the `--buildpack-path` argument to indicate the specific location of the buildpack to use. |
+| `BP_ARCH`             | `runtime.GOARCH` (i.e. your system's arch) | This does not generally need to be set, but you can use it to override the automatically detected architecture. This might be helpful if you're on M-series Mac hardware and can build for multiple architectures.                                                                                                                                                                                                                                                       |
+| `BP_PULL_POLICY`      | `if-not-present`                           | This will allow you to override the pull policy. The tool specifically sets pull policy, and does not default to pack's default.                                                                                                                                                                                                                                                                                                                                         |
+| `BP_FLATTEN_DISABLED` | `false`                                    | This will disable flattening of composite buildpacks. By default, the tool will flatten composite buildpacks which takes all of the component buildpacks in that composite buildpack and puts them into one layer, instead of many layers.                                                                                                                                                                                                                               |
+
 ## `libpak-tools package compile`
 
 The `package compile` command creates a `libpak.Package` and calls `libpak.Package.Create()`. This takes a Paketo buildpack written in Go and packages is it into a buildpack. That involves compiling the source code, possibly copying in additional resource files, and generating the buildpack in the given output directory. The key is that the output of this command is a *directory*. If you want it to output an image, use `libpak-tools package bundle`.
@@ -29,8 +38,7 @@ Flags:
 The `package bundle` does the same thing as `libpak-tools package compile` but then runs `pack buildpack package` as well, so the output is a buildpack image.
 
 ```
-> libpak-tools package bundle -h
-Compile and package a single buildpack
+Compile and package a single buildpack (component & composite)
 
 Usage:
   libpak-tools package bundle [flags]
@@ -42,6 +50,8 @@ Flags:
       --dependency-filter stringArray   one or more filters that are applied to exclude dependencies
   -h, --help                            help for bundle
       --include-dependencies            whether to include dependencies (default: false)
+      --publish                         publish the buildpack to a buildpack registry (default: false)
+      --registry-name string            prefix for the registry to publish to (default: your buildpack id)
       --strict-filters                  require filter to match all data or just some data (default: false)
       --version string                  version to substitute into buildpack.toml/extension.toml
 ```

commands/package_buildpack.go

@@ -29,7 +29,7 @@ func PackageBundleCommand() *cobra.Command {
 
 	var packageBuildpackCmd = &cobra.Command{
 		Use:   "bundle",
-		Short: "Compile and package a single buildpack",
+		Short: "Compile and package a single buildpack (component & composite)",
 		Run: func(cmd *cobra.Command, args []string) {
 			if p.BuildpackID == "" && p.BuildpackPath == "" {
 				log.Fatal("buildpack-id or buildpack-path must be set")
@@ -40,11 +40,15 @@ func PackageBundleCommand() *cobra.Command {
 			}
 
 			if p.BuildpackID != "" && p.BuildpackPath == "" {
-				p.InferBuildpackPath()
+				if err := p.InferBuildpackPath(); err != nil {
+					log.Fatal(err)
+				}
 			}
 
 			if p.BuildpackVersion == "" {
-				p.InferBuildpackVersion()
+				if err := p.InferBuildpackVersion(); err != nil {
+					log.Fatal(err)
+				}
 			}
 
 			if p.RegistryName == "" {

packager/buildpack.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/buildpacks/libcnb/v2"
 	"github.com/paketo-buildpacks/libpak/v2/effect"
+	"github.com/paketo-buildpacks/libpak/v2/sherpa"
 
 	"github.com/paketo-buildpacks/libpak-tools/carton"
 )
@@ -167,15 +168,15 @@ func (p *BundleBuildpack) CleanUpDockerImages() error {
 			Stderr: io.Discard,
 		})
 		if err != nil {
-			return fmt.Errorf("unable to execute `docker image rm` command\n%w", err)
+			return fmt.Errorf("unable to execute `docker image rm` command on images %v\n%w", imagesToClean, err)
 		}
 	}
 
 	return nil
 }
 
 // ExecutePackage runs the package buildpack command
-func (p *BundleBuildpack) ExecutePackage(tmpDir string) error {
+func (p *BundleBuildpack) ExecutePackage(workingDirectory string, additionalArgs ...string) error {
 	pullPolicy, found := os.LookupEnv("BP_PULL_POLICY")
 	if !found {
 		pullPolicy = "if-not-present"
@@ -199,12 +200,13 @@ func (p *BundleBuildpack) ExecutePackage(tmpDir string) error {
 		args = append(args, "--target", archFromSystem())
 	}
 
+	args = append(args, additionalArgs...)
 	err := p.executor.Execute(effect.Execution{
 		Command: "pack",
 		Args:    args,
 		Stdout:  os.Stdout,
 		Stderr:  os.Stderr,
-		Dir:     tmpDir,
+		Dir:     workingDirectory,
 	})
 	if err != nil {
 		return fmt.Errorf("unable to execute `pack buildpack package` command\n%w", err)
@@ -214,15 +216,15 @@ func (p *BundleBuildpack) ExecutePackage(tmpDir string) error {
 }
 
 // CompilePackage compiles the buildpack's Go code
-func (p *BundleBuildpack) CompilePackage(tmpDir string) {
+func (p *BundleBuildpack) CompilePackage(destDir string) {
 	pkg := carton.Package{}
 	pkg.Source = p.BuildpackPath
 	pkg.Version = p.BuildpackVersion
 	pkg.CacheLocation = p.CacheLocation
 	pkg.DependencyFilters = p.DependencyFilters
 	pkg.StrictDependencyFilters = p.StrictDependencyFilters
 	pkg.IncludeDependencies = p.IncludeDependencies
-	pkg.Destination = tmpDir
+	pkg.Destination = destDir
 
 	options := []carton.Option{
 		carton.WithExecutor(p.executor),
@@ -233,22 +235,79 @@ func (p *BundleBuildpack) CompilePackage(tmpDir string) {
 	pkg.Create(options...)
 }
 
-// Execute runs the package buildpack command
-func (p *BundleBuildpack) Execute() error {
-	tmpDir, err := os.MkdirTemp("", "BundleBuildpack")
-	if err != nil {
-		return fmt.Errorf("unable to create temporary directory\n%w", err)
-	}
-
+func (p *BundleBuildpack) CompileAndBundleComponent(buildDirectory string) error {
 	// Compile the buildpack
 	fmt.Println("➜ Compile Buildpack")
-	p.CompilePackage(tmpDir)
+	p.CompilePackage(buildDirectory)
 
 	// package the buildpack
 	fmt.Printf("➜ Package Buildpack: %s\n", p.BuildpackID)
-	err = p.ExecutePackage(tmpDir)
+	return p.ExecutePackage(buildDirectory)
+}
+
+func (p *BundleBuildpack) BundleComposite(buildDirectory string) error {
+	// Make a modified package.toml in the temp directory
+	packageTomlPath, err := copyPackageTomlAndAddURI(p.BuildpackPath, buildDirectory)
+	if err != nil {
+		return fmt.Errorf("unable to copy package.toml and add URI\n%w", err)
+	}
+
+	// prepare extra arguments
+	args := []string{
+		"--config", packageTomlPath,
+	}
+
+	if !sherpa.ResolveBool("BP_FLATTEN_DISABLED") {
+		args = append(args, "--flatten")
+	}
+
+	// we still package from the buildpack directory though, only the package.toml is in the temp directory
+	fmt.Printf("➜ Package Buildpack: %s\n", p.BuildpackID)
+	return p.ExecutePackage(p.BuildpackPath, args...)
+}
+
+func copyPackageTomlAndAddURI(buildpackPath, destDir string) (string, error) {
+	inputPackageToml, err := os.Open(filepath.Join(buildpackPath, "package.toml"))
 	if err != nil {
-		return fmt.Errorf("unable to package the buildpack\n%w", err)
+		return "", fmt.Errorf("unable to open package.toml\n%w", err)
+	}
+	defer inputPackageToml.Close()
+
+	outputPackageTomlPath := filepath.Join(destDir, "package.toml")
+	outputPackageToml, err := os.Create(outputPackageTomlPath)
+	if err != nil {
+		return "", fmt.Errorf("unable to create package.toml\n%w", err)
+	}
+	defer outputPackageToml.Close()
+
+	_, err = outputPackageToml.WriteString(fmt.Sprintf("[buildpack]\nuri = \"%s\"\n\n", buildpackPath))
+	if err != nil {
+		return "", fmt.Errorf("unable to write uri\n%w", err)
+	}
+
+	_, err = io.Copy(outputPackageToml, inputPackageToml)
+	if err != nil {
+		return "", fmt.Errorf("unable to copy rest of package.toml\n%w", err)
+	}
+
+	return outputPackageTomlPath, nil
+}
+
+// Execute runs the package buildpack command
+func (p *BundleBuildpack) Execute() error {
+	buildDirectory, err := os.MkdirTemp("", "BundleBuildpack")
+	if err != nil {
+		return fmt.Errorf("unable to create temporary directory\n%w", err)
+	}
+
+	// we use existence of main.go to determine if we are packaging a component or composite buildpack
+	mainCmdPath := filepath.Join(p.BuildpackPath, "cmd/main/main.go")
+	if componentBp, err := sherpa.FileExists(mainCmdPath); err != nil {
+		return fmt.Errorf("unable to check if file exists\n%w", err)
+	} else if componentBp {
+		p.CompileAndBundleComponent(buildDirectory)
+	} else {
+		p.BundleComposite(buildDirectory)
 	}
 
 	// clean up

packager/buildpack_test.go

@@ -18,6 +18,7 @@ package packager_test
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"testing"
 
 	exMocks "github.com/buildpacks/libcnb/v2/mocks"
@@ -35,6 +36,10 @@ func testBuildpack(t *testing.T, context spec.G, it spec.S) {
 		Expect = NewWithT(t).Expect
 	)
 
+	it.Before(func() {
+		t.Setenv("BP_ARCH", "amd64")
+	})
+
 	context("Infer Buildpack Path", func() {
 		context("BP_ROOT is not set", func() {
 			it.Before(func() {
@@ -246,8 +251,6 @@ func testBuildpack(t *testing.T, context spec.G, it spec.S) {
 
 		it.Before(func() {
 			mockExecutor = &mocks.Executor{}
-
-			t.Setenv("BP_ARCH", "amd64")
 		})
 
 		it("defaults pull policy to if-not-present", func() {
@@ -313,6 +316,26 @@ func testBuildpack(t *testing.T, context spec.G, it spec.S) {
 				Expect(p.ExecutePackage("/some/path")).To(Succeed())
 			})
 		})
+
+		it("includes additional args", func() {
+			mockExecutor.On("Execute", mock.MatchedBy(func(e effect.Execution) bool {
+				return e.Command == "pack" &&
+					e.Args[0] == "buildpack" &&
+					e.Args[1] == "package" &&
+					e.Args[2] == "some-id" &&
+					e.Args[3] == "--pull-policy" &&
+					e.Args[4] == "if-not-present" &&
+					e.Args[5] == "--target" &&
+					e.Args[6] == "linux/amd64" &&
+					e.Args[7] == "--some-more-args" &&
+					e.Dir == "/some/path"
+			})).Return(nil)
+
+			p := packager.NewBundleBuildpackForTests(mockExecutor, nil)
+			p.BuildpackID = "some-id"
+
+			Expect(p.ExecutePackage("/some/path", "--some-more-args")).To(Succeed())
+		})
 	})
 
 	context("CompilePackage", func() {
@@ -347,4 +370,52 @@ func testBuildpack(t *testing.T, context spec.G, it spec.S) {
 			Expect(p.ExecutePackage("/some/path")).To(Succeed())
 		})
 	})
+
+	context("Bundles a Composite", func() {
+		var (
+			buildpackPath string
+			buildPath     string
+			mockExecutor  *mocks.Executor
+		)
+
+		it.Before(func() {
+			buildpackPath = t.TempDir()
+			buildPath = t.TempDir()
+			mockExecutor = &mocks.Executor{}
+
+			Expect(os.WriteFile(filepath.Join(buildpackPath, "package.toml"), []byte("some-toml"), 0600)).To(Succeed())
+		})
+
+		it("inserts the URI to package.toml and runs pack buildpack package", func() {
+			mockExecutor.On("Execute", mock.MatchedBy(func(e effect.Execution) bool {
+				Expect(e.Command).To(Equal("pack"))
+				Expect(e.Args).To(HaveExactElements([]string{
+					"buildpack",
+					"package",
+					"some-id",
+					"--pull-policy",
+					"if-not-present",
+					"--target",
+					"linux/amd64",
+					"--config",
+					filepath.Join(buildPath, "/package.toml"),
+					"--flatten",
+				}))
+				Expect(e.Dir).To(Equal(buildpackPath))
+				return true
+			})).Return(nil)
+
+			p := packager.NewBundleBuildpackForTests(mockExecutor, nil)
+			p.BuildpackID = "some-id"
+			p.BuildpackPath = buildpackPath
+
+			Expect(p.BundleComposite(buildPath)).To(Succeed())
+
+			packageToml := filepath.Join(buildPath, "package.toml")
+			Expect(packageToml).To(BeARegularFile())
+			contents, err := os.ReadFile(packageToml)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(string(contents)).To(HavePrefix(fmt.Sprintf("[buildpack]\nuri = \"%s\"\n\n", buildpackPath)))
+		})
+	})
 }