Dockerfiles phase 2

Makefile

@@ -118,6 +118,7 @@ build-linux-amd64-symlinks:
 	ln -sf lifecycle $(OUT_DIR)/exporter
 	ln -sf lifecycle $(OUT_DIR)/rebaser
 	ln -sf lifecycle $(OUT_DIR)/creator
+	ln -sf lifecycle $(OUT_DIR)/extender
 
 build-linux-arm64-symlinks: export GOOS:=linux
 build-linux-arm64-symlinks: export GOARCH:=arm64
@@ -131,6 +132,7 @@ build-linux-arm64-symlinks:
 	ln -sf lifecycle $(OUT_DIR)/exporter
 	ln -sf lifecycle $(OUT_DIR)/rebaser
 	ln -sf lifecycle $(OUT_DIR)/creator
+	ln -sf lifecycle $(OUT_DIR)/extender
 
 build-windows-amd64-lifecycle: $(BUILD_DIR)/windows-amd64/lifecycle/lifecycle.exe
 

acceptance/detector_test.go

@@ -350,8 +350,8 @@ func testDetector(t *testing.T, when spec.G, it spec.S) {
 				h.WithArgs(
 					"-analyzed=/layers/analyzed.toml",
 					"-extensions=/cnb/extensions",
-					"-log-level=debug",
 					"-generated=/layers/generated",
+					"-log-level=debug",
 				),
 			)
 
@@ -374,9 +374,7 @@ func testDetector(t *testing.T, when spec.G, it spec.S) {
 			var plan platform.BuildPlan
 			_, err = toml.DecodeFile(foundPlanTOML, &plan)
 			h.AssertNil(t, err)
-			h.AssertEq(t, plan.Entries[0].Requires[0].Name, "some_requirement")
-			h.AssertEq(t, plan.Entries[0].Providers[0].ID, "simple_extension")
-			h.AssertEq(t, plan.Entries[0].Providers[0].Extension, true)
+			h.AssertEq(t, len(plan.Entries), 0) // this shows that the plan was filtered to remove `requires` provided by extensions
 
 			t.Log("runs /bin/generate for extensions")
 			h.AssertStringContains(t, output, "simple_extension: output from /bin/generate")

acceptance/extender_test.go

@@ -0,0 +1,146 @@
+//go:build acceptance
+// +build acceptance
+
+package acceptance
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/sclevine/spec"
+	"github.com/sclevine/spec/report"
+
+	"github.com/buildpacks/lifecycle/api"
+	"github.com/buildpacks/lifecycle/auth"
+	"github.com/buildpacks/lifecycle/internal/selective"
+	h "github.com/buildpacks/lifecycle/testhelpers"
+)
+
+var (
+	extendImage          string
+	extendRegAuthConfig  string
+	extendRegNetwork     string
+	extenderPath         string
+	extendDaemonFixtures *daemonImageFixtures
+	extendRegFixtures    *regImageFixtures
+	extendTest           *PhaseTest
+)
+
+func TestExtender(t *testing.T) {
+	h.SkipIf(t, runtime.GOOS == "windows", "Extender is not supported on Windows")
+
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	testImageDockerContext := filepath.Join("testdata", "extender")
+	extendTest = NewPhaseTest(t, "extender", testImageDockerContext)
+	extendTest.Start(t)
+	defer extendTest.Stop(t)
+
+	extendImage = extendTest.testImageRef
+	extenderPath = extendTest.containerBinaryPath
+	extendRegAuthConfig = extendTest.targetRegistry.authConfig
+	extendRegNetwork = extendTest.targetRegistry.network
+	extendDaemonFixtures = extendTest.targetDaemon.fixtures
+	extendRegFixtures = extendTest.targetRegistry.fixtures
+
+	for _, platformAPI := range api.Platform.Supported {
+		spec.Run(t, "acceptance-extender/"+platformAPI.String(), testExtenderFunc(platformAPI.String()), spec.Parallel(), spec.Report(report.Terminal{}))
+	}
+}
+
+func testExtenderFunc(platformAPI string) func(t *testing.T, when spec.G, it spec.S) {
+	return func(t *testing.T, when spec.G, it spec.S) {
+		it.Before(func() {
+			h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.10"), "")
+		})
+
+		when("kaniko case", func() {
+			var kanikoDir, buildImageDigest string
+
+			it.Before(func() {
+				var err error
+				kanikoDir, err = ioutil.TempDir("", "lifecycle-acceptance")
+				h.AssertNil(t, err)
+
+				// push "builder" image to test registry
+				h.Run(t, exec.Command("docker", "tag", extendImage, extendTest.RegRepoName(extendImage)))
+				h.AssertNil(t, h.PushImage(h.DockerCli(t), extendTest.RegRepoName(extendImage), extendTest.targetRegistry.registry.EncodedLabeledAuth()))
+
+				// warm kaniko cache - this mimics what the analyzer or restorer would have done
+				os.Setenv("DOCKER_CONFIG", extendTest.targetRegistry.dockerConfigDir)
+				ref, auth, err := auth.ReferenceForRepoName(authn.DefaultKeychain, extendTest.RegRepoName(extendImage))
+				h.AssertNil(t, err)
+				remoteImage, err := remote.Image(ref, remote.WithAuth(auth))
+				h.AssertNil(t, err)
+				buildImageHash, err := remoteImage.Digest()
+				h.AssertNil(t, err)
+				buildImageDigest = buildImageHash.String()
+				baseCacheDir := filepath.Join(kanikoDir, "cache", "base")
+				h.AssertNil(t, os.MkdirAll(baseCacheDir, 0755))
+				layoutPath, err := selective.Write(filepath.Join(baseCacheDir, buildImageDigest), empty.Index)
+				h.AssertNil(t, err)
+				h.AssertNil(t, layoutPath.AppendImage(remoteImage))
+			})
+
+			it.After(func() {
+				_ = os.RemoveAll(kanikoDir)
+			})
+
+			when("extending the build image", func() {
+				it("succeeds", func() {
+					extendArgs := []string{
+						ctrPath(extenderPath),
+						"-generated", "/layers/generated",
+						"-log-level", "debug",
+						"-gid", "1000",
+						"-uid", "1234",
+						"oci:/kaniko/cache/base/" + buildImageDigest,
+					}
+
+					t.Log("first build extends the build image by running Dockerfile commands")
+					firstOutput := h.DockerRunWithCombinedOutput(t,
+						extendImage,
+						h.WithFlags(
+							"--env", "CNB_PLATFORM_API="+platformAPI,
+							"--volume", fmt.Sprintf("%s:/kaniko", kanikoDir),
+						),
+						h.WithArgs(extendArgs...),
+					)
+					h.AssertStringDoesNotContain(t, firstOutput, "Did not find cache key, pulling remote image...")
+					h.AssertStringContains(t, firstOutput, "ca-certificates")
+					h.AssertStringContains(t, firstOutput, "Hello Extensions buildpack\ncurl") // output by buildpack, shows that curl was installed on the build image
+					t.Log("sets environment variables from the extended build image in the build context")
+					h.AssertStringContains(t, firstOutput, "CNB_STACK_ID for buildpack: stack-id-from-ext-tree")
+
+					t.Log("cleans the kaniko directory")
+					fis, err := ioutil.ReadDir(kanikoDir)
+					h.AssertNil(t, err)
+					h.AssertEq(t, len(fis), 1) // 1: /kaniko/cache
+
+					t.Log("second build extends the build image by pulling from the cache directory")
+					secondOutput := h.DockerRunWithCombinedOutput(t,
+						extendImage,
+						h.WithFlags(
+							"--env", "CNB_PLATFORM_API="+platformAPI,
+							"--volume", fmt.Sprintf("%s:/kaniko", kanikoDir),
+						),
+						h.WithArgs(extendArgs...),
+					)
+					h.AssertStringDoesNotContain(t, secondOutput, "Did not find cache key, pulling remote image...")
+					h.AssertStringDoesNotContain(t, secondOutput, "ca-certificates")
+					h.AssertStringContains(t, secondOutput, "Hello Extensions buildpack\ncurl") // output by buildpack, shows that curl is still installed in the unpacked cached layer
+				})
+			})
+		})
+	}
+}

acceptance/restorer_test.go

@@ -21,9 +21,13 @@ import (
 )
 
 var (
-	restoreDockerContext = filepath.Join("testdata", "restorer")
-	restorerBinaryDir    = filepath.Join("testdata", "restorer", "container", "cnb", "lifecycle")
-	restorerImage        = "lifecycle/acceptance/restorer"
+	restoreImage          string
+	restoreRegAuthConfig  string
+	restoreRegNetwork     string
+	restorerPath          string
+	restoreDaemonFixtures *daemonImageFixtures
+	restoreRegFixtures    *regImageFixtures
+	restoreTest           *PhaseTest
 )
 
 func TestRestorer(t *testing.T) {
@@ -32,9 +36,17 @@ func TestRestorer(t *testing.T) {
 
 	rand.Seed(time.Now().UTC().UnixNano())
 
-	h.MakeAndCopyLifecycle(t, "linux", "amd64", restorerBinaryDir)
-	h.DockerBuild(t, restorerImage, restoreDockerContext)
-	defer h.DockerImageRemove(t, restorerImage)
+	testImageDockerContext := filepath.Join("testdata", "restorer")
+	restoreTest = NewPhaseTest(t, "restorer", testImageDockerContext)
+	restoreTest.Start(t)
+	defer restoreTest.Stop(t)
+
+	restoreImage = restoreTest.testImageRef
+	restorerPath = restoreTest.containerBinaryPath
+	restoreRegAuthConfig = restoreTest.targetRegistry.authConfig
+	restoreRegNetwork = restoreTest.targetRegistry.network
+	restoreDaemonFixtures = restoreTest.targetDaemon.fixtures
+	restoreRegFixtures = restoreTest.targetRegistry.fixtures
 
 	for _, platformAPI := range api.Platform.Supported {
 		spec.Run(t, "acceptance-restorer/"+platformAPI.String(), testRestorerFunc(platformAPI.String()), spec.Parallel(), spec.Report(report.Terminal{}))
@@ -43,9 +55,24 @@ func TestRestorer(t *testing.T) {
 
 func testRestorerFunc(platformAPI string) func(t *testing.T, when spec.G, it spec.S) {
 	return func(t *testing.T, when spec.G, it spec.S) {
+		var copyDir, containerName string
+		it.Before(func() {
+			containerName = "test-container-" + h.RandString(10)
+			var err error
+			copyDir, err = ioutil.TempDir("", "test-docker-copy-")
+			h.AssertNil(t, err)
+		})
+
+		it.After(func() {
+			if h.DockerContainerExists(t, containerName) {
+				h.Run(t, exec.Command("docker", "rm", containerName))
+			}
+			_ = os.RemoveAll(copyDir)
+		})
+
 		when("called with arguments", func() {
 			it("errors", func() {
-				command := exec.Command("docker", "run", "--rm", restorerImage, "some-arg")
+				command := exec.Command("docker", "run", "--rm", restoreImage, "some-arg")
 				output, err := command.CombinedOutput()
 				h.AssertNotNil(t, err)
 				expected := "failed to parse arguments: received unexpected Args"
@@ -56,7 +83,7 @@ func testRestorerFunc(platformAPI string) func(t *testing.T, when spec.G, it spe
 		when("called with -analyzed", func() {
 			it("errors", func() {
 				h.SkipIf(t, api.MustParse(platformAPI).AtLeast("0.7"), "Platform API >= 0.7 supports -analyzed flag")
-				command := exec.Command("docker", "run", "--rm", restorerImage, "-analyzed some-file-location")
+				command := exec.Command("docker", "run", "--rm", restoreImage, "-analyzed some-file-location")
 				output, err := command.CombinedOutput()
 				h.AssertNotNil(t, err)
 				expected := "flag provided but not defined: -analyzed"
@@ -67,7 +94,7 @@ func testRestorerFunc(platformAPI string) func(t *testing.T, when spec.G, it spe
 		when("called with -skip-layers", func() {
 			it("errors", func() {
 				h.SkipIf(t, api.MustParse(platformAPI).AtLeast("0.7"), "Platform API >= 0.7 supports -skip-layers flag")
-				command := exec.Command("docker", "run", "--rm", restorerImage, "-skip-layers true")
+				command := exec.Command("docker", "run", "--rm", restoreImage, "-skip-layers true")
 				output, err := command.CombinedOutput()
 				h.AssertNotNil(t, err)
 				expected := "flag provided but not defined: -skip-layers"
@@ -77,7 +104,7 @@ func testRestorerFunc(platformAPI string) func(t *testing.T, when spec.G, it spe
 
 		when("called without any cache flag", func() {
 			it("outputs it will not restore cache layer data", func() {
-				command := exec.Command("docker", "run", "--rm", "--env", "CNB_PLATFORM_API="+platformAPI, restorerImage)
+				command := exec.Command("docker", "run", "--rm", "--env", "CNB_PLATFORM_API="+platformAPI, restoreImage)
 				output, err := command.CombinedOutput()
 				h.AssertNil(t, err)
 				expected := "Not restoring cached layer data, no cache flag specified"
@@ -86,28 +113,13 @@ func testRestorerFunc(platformAPI string) func(t *testing.T, when spec.G, it spe
 		})
 
 		when("analyzed.toml exists with app metadata", func() {
-			var copyDir, containerName string
-			it.Before(func() {
-				containerName = "test-container-" + h.RandString(10)
-				var err error
-				copyDir, err = ioutil.TempDir("", "test-docker-copy-")
-				h.AssertNil(t, err)
-			})
-
-			it.After(func() {
-				if h.DockerContainerExists(t, containerName) {
-					h.Run(t, exec.Command("docker", "rm", containerName))
-				}
-				os.RemoveAll(copyDir)
-			})
-
 			it("restores app metadata", func() {
 				h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.7"), "Platform API < 0.7 does not restore app metadata")
 				output := h.DockerRunAndCopy(t,
 					containerName,
 					copyDir,
 					ctrPath("/layers"),
-					restorerImage,
+					restoreImage,
 					h.WithFlags(append(
 						dockerSocketMount,
 						"--env", "CNB_PLATFORM_API="+platformAPI,
@@ -117,33 +129,16 @@ func testRestorerFunc(platformAPI string) func(t *testing.T, when spec.G, it spe
 
 				h.AssertStringContains(t, output, "Restoring metadata for \"some-buildpack-id:launch-layer\"")
 			})
-
 		})
 
 		when("using cache-dir", func() {
 			when("there is cache present from a previous build", func() {
-				var copyDir, containerName string
-
-				it.Before(func() {
-					containerName = "test-container-" + h.RandString(10)
-					var err error
-					copyDir, err = ioutil.TempDir("", "test-docker-copy-")
-					h.AssertNil(t, err)
-				})
-
-				it.After(func() {
-					if h.DockerContainerExists(t, containerName) {
-						h.Run(t, exec.Command("docker", "rm", containerName))
-					}
-					os.RemoveAll(copyDir)
-				})
-
 				it("restores cached layer data", func() {
 					h.DockerRunAndCopy(t,
 						containerName,
 						copyDir,
 						"/layers",
-						restorerImage,
+						restoreImage,
 						h.WithFlags("--env", "CNB_PLATFORM_API="+platformAPI),
 						h.WithArgs("-cache-dir", "/cache"),
 					)
@@ -163,7 +158,7 @@ func testRestorerFunc(platformAPI string) func(t *testing.T, when spec.G, it spe
 						containerName,
 						copyDir,
 						"/layers",
-						restorerImage,
+						restoreImage,
 						h.WithFlags("--env", "CNB_PLATFORM_API="+platformAPI),
 						h.WithArgs("-cache-dir", "/cache"),
 					)
@@ -181,7 +176,7 @@ func testRestorerFunc(platformAPI string) func(t *testing.T, when spec.G, it spe
 						containerName,
 						copyDir,
 						"/layers",
-						restorerImage,
+						restoreImage,
 						h.WithFlags("--env", "CNB_PLATFORM_API="+platformAPI),
 						h.WithArgs("-cache-dir", "/cache"),
 					)
@@ -192,5 +187,28 @@ func testRestorerFunc(platformAPI string) func(t *testing.T, when spec.G, it spe
 				})
 			})
 		})
+
+		when("using kaniko cache", func() {
+			it("accepts -build-image", func() {
+				h.SkipIf(t, api.MustParse(platformAPI).LessThan("0.10"), "Platform API < 0.10 does not use kaniko")
+				h.DockerRunAndCopy(t,
+					containerName,
+					copyDir,
+					"/kaniko",
+					restoreImage,
+					h.WithFlags(
+						"--env", "CNB_PLATFORM_API="+platformAPI,
+						"--env", "DOCKER_CONFIG=/docker-config",
+						"--network", restoreRegNetwork,
+					),
+					h.WithArgs("-build-image", restoreRegFixtures.SomeCacheImage), // some-cache-image simulates a builder image in a registry
+				)
+				t.Log("writes builder manifest and config to the kaniko cache")
+				fis, err := os.ReadDir(filepath.Join(copyDir, "kaniko", "cache", "base"))
+				h.AssertNil(t, err)
+				h.AssertEq(t, len(fis), 1)
+				h.AssertPathExists(t, filepath.Join(copyDir, "kaniko", "cache", "base", fis[0].Name(), "oci-layout"))
+			})
+		})
 	}
 }

acceptance/testdata/extender/Dockerfile

@@ -0,0 +1,3 @@
+FROM ubuntu:bionic
+
+COPY ./container/ /