diff --git a/.github/workflows/intergration-layers-tests.yaml b/.github/workflows/integration-layers-tests.yaml similarity index 96% rename from .github/workflows/intergration-layers-tests.yaml rename to .github/workflows/integration-layers-tests.yaml index 3ba9b05501..df705d8f8c 100644 --- a/.github/workflows/intergration-layers-tests.yaml +++ b/.github/workflows/integration-layers-tests.yaml @@ -28,4 +28,5 @@ jobs: - name: Run integration-test-layers run: | make travis-setup + make minikube-setup make integration-test-layers diff --git a/.github/workflows/integration-mis.yaml b/.github/workflows/integration-misc.yaml similarity index 97% rename from .github/workflows/integration-mis.yaml rename to .github/workflows/integration-misc.yaml index ca7f77191d..0e928202c7 100644 --- a/.github/workflows/integration-mis.yaml +++ b/.github/workflows/integration-misc.yaml @@ -37,4 +37,5 @@ jobs: - name: Run integration-test-misc run : | make travis-setup + make minikube-setup make integration-test-misc \ No newline at end of file diff --git a/.github/workflows/integration-run-tests.yaml b/.github/workflows/integration-run-tests.yaml index 4138cd8fba..62921e8cb6 100644 --- a/.github/workflows/integration-run-tests.yaml +++ b/.github/workflows/integration-run-tests.yaml @@ -27,4 +27,5 @@ jobs: - name: Run integration-test-run run: | make travis-setup + make minikube-setup make integration-test-run \ No newline at end of file diff --git a/.github/workflows/integration_k8_tests.yaml b/.github/workflows/integration_k8_tests.yaml index d671b41949..d21c8a248d 100644 --- a/.github/workflows/integration_k8_tests.yaml +++ b/.github/workflows/integration_k8_tests.yaml @@ -1,6 +1,6 @@ -name: Intergration tests (K8s) +name: Integration tests (K8s) -# Triggers the workflow on push or pull request events +# Triggers the workflow on pull request events on: [pull_request] concurrency: diff --git a/integration/benchmark_test.go b/integration/benchmark_test.go index f7364114d6..4451f54ebc 100644 --- a/integration/benchmark_test.go +++ b/integration/benchmark_test.go @@ -59,7 +59,7 @@ func TestSnapshotBenchmark(t *testing.T) { kanikoImage := fmt.Sprintf("%s_%d", GetKanikoImage(config.imageRepo, dockerfile), num) buildArgs := []string{"--build-arg", fmt.Sprintf("NUM=%d", num)} var benchmarkDir string - benchmarkDir, *err = buildKanikoImage("", dockerfile, + benchmarkDir, *err = buildKanikoImage(t.Logf, "", dockerfile, buildArgs, []string{}, kanikoImage, contextDir, config.gcsBucket, config.serviceAccount, false) if *err != nil { @@ -77,11 +77,11 @@ func TestSnapshotBenchmark(t *testing.T) { } wg.Wait() - fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files") + t.Log("Number of Files,Total Build Time,Walking Filesystem, Resolving Files") timeMap.Range(func(key interface{}, value interface{}) bool { d, _ := key.(int) v, _ := value.(result) - fmt.Println(fmt.Sprintf("%d,%f,%f,%f", d, v.totalBuildTime, v.walkingFiles, v.resolvingFiles)) + t.Logf("%d,%f,%f,%f", d, v.totalBuildTime, v.walkingFiles, v.resolvingFiles) return true }) @@ -111,7 +111,7 @@ func newResult(t *testing.T, f string) result { if c, ok := current["Hashing files"]; ok { r.hashingFiles = c.Seconds() } - fmt.Println(r) + t.Log(r) return r } @@ -128,7 +128,7 @@ func TestSnapshotBenchmarkGcloud(t *testing.T) { nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000} var wg sync.WaitGroup - fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files") + t.Log("Number of Files,Total Build Time,Walking Filesystem, Resolving Files") for _, num := range nums { t.Run(fmt.Sprintf("test_benchmark_%d", num), func(t *testing.T) { wg.Add(1) @@ -139,7 +139,7 @@ func TestSnapshotBenchmarkGcloud(t *testing.T) { return } r := newResult(t, filepath.Join(dir, "results")) - fmt.Println(fmt.Sprintf("%d,%f,%f,%f, %f", num, r.totalBuildTime, r.walkingFiles, r.resolvingFiles, r.hashingFiles)) + t.Log(fmt.Sprintf("%d,%f,%f,%f, %f", num, r.totalBuildTime, r.walkingFiles, r.resolvingFiles, r.hashingFiles)) wg.Done() defer os.Remove(dir) defer os.Chdir(cwd) diff --git a/integration/images.go b/integration/images.go index 958d3b4bca..1848904d69 100644 --- a/integration/images.go +++ b/integration/images.go @@ -27,6 +27,7 @@ import ( "runtime" "strconv" "strings" + "testing" "time" "github.com/GoogleContainerTools/kaniko/pkg/timing" @@ -183,10 +184,14 @@ type DockerFileBuilder struct { TestCacheDockerfiles map[string]struct{} } +type logger func(string, ...interface{}) + // NewDockerFileBuilder will create a DockerFileBuilder initialized with dockerfiles, which // it will assume are all as yet unbuilt. func NewDockerFileBuilder() *DockerFileBuilder { - d := DockerFileBuilder{filesBuilt: map[string]struct{}{}} + d := DockerFileBuilder{ + filesBuilt: map[string]struct{}{}, + } d.DockerfilesToIgnore = map[string]struct{}{ "Dockerfile_test_add_404": {}, // TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed @@ -218,8 +223,8 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string { return flags } -func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir string) error { - fmt.Printf("Building image for Dockerfile %s\n", dockerfile) +func (d *DockerFileBuilder) BuildDockerImage(t *testing.T, imageRepo, dockerfilesPath, dockerfile, contextDir string) error { + t.Logf("Building image for Dockerfile %s\n", dockerfile) var buildArgs []string buildArgFlag := "--build-arg" @@ -252,21 +257,21 @@ func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerf if err != nil { return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out)) } - fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out) + t.Logf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out) return nil } // BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. // The resulting image will be tagged with imageRepo. If the dockerfile will be built with // context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. -func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error { +func (d *DockerFileBuilder) BuildImage(t *testing.T, config *integrationTestConfig, dockerfilesPath, dockerfile string) error { _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) - return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd) + return d.BuildImageWithContext(t, config, dockerfilesPath, dockerfile, cwd) } -func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error { +func (d *DockerFileBuilder) BuildImageWithContext(t *testing.T, config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error { if _, present := d.filesBuilt[dockerfile]; present { return nil } @@ -279,7 +284,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, } timer := timing.Start(dockerfile + "_docker") - d.BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir) + d.BuildDockerImage(t, imageRepo, dockerfilesPath, dockerfile, contextDir) timing.DefaultRun.Stop(timer) contextFlag := "-c" @@ -302,7 +307,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, kanikoImage := GetKanikoImage(imageRepo, dockerfile) timer = timing.Start(dockerfile + "_kaniko") - if _, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage, + if _, err := buildKanikoImage(t.Logf, dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage, contextDir, gcsBucket, serviceAccount, true); err != nil { return err } @@ -420,6 +425,7 @@ func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, servi } func buildKanikoImage( + logf logger, dockerfilesPath string, dockerfile string, buildArgs []string, @@ -447,7 +453,7 @@ func buildKanikoImage( // build kaniko image additionalFlags := append(buildArgs, kanikoArgs...) - fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags) + logf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags) dockerRunFlags := []string{"run", "--net=host", "-e", benchmarkEnv, diff --git a/integration/integration_test.go b/integration/integration_test.go index 96924f0984..cdfe7a72d3 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -87,8 +87,8 @@ func getDockerMajorVersion() int { } return ver } -func launchTests(m *testing.M) (int, error) { +func launchTests(m *testing.M) (int, error) { if config.isGcrRepository() { contextFile, err := CreateIntegrationTarball() if err != nil { @@ -141,32 +141,25 @@ func buildRequiredImages() error { setupCommands := []struct { name string command []string - }{ - { - name: "Building kaniko image", - command: []string{"docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", ".."}, - }, - { - name: "Building cache warmer image", - command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."}, - }, - { - name: "Building onbuild base image", - command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."}, - }, - { - name: "Pushing onbuild base image", - command: []string{"docker", "push", config.onbuildBaseImage}, - }, - { - name: "Building hardlink base image", - command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."}, - }, - { - name: "Pushing hardlink base image", - command: []string{"docker", "push", config.hardlinkBaseImage}, - }, - } + }{{ + name: "Building kaniko image", + command: []string{"docker", "build", "-t", ExecutorImage, "-f", "../deploy/Dockerfile", ".."}, + }, { + name: "Building cache warmer image", + command: []string{"docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", ".."}, + }, { + name: "Building onbuild base image", + command: []string{"docker", "build", "-t", config.onbuildBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_onbuild_base", dockerfilesPath), "."}, + }, { + name: "Pushing onbuild base image", + command: []string{"docker", "push", config.onbuildBaseImage}, + }, { + name: "Building hardlink base image", + command: []string{"docker", "build", "-t", config.hardlinkBaseImage, "-f", fmt.Sprintf("%s/Dockerfile_hardlink_base", dockerfilesPath), "."}, + }, { + name: "Pushing hardlink base image", + command: []string{"docker", "push", config.hardlinkBaseImage}, + }} for _, setupCmd := range setupCommands { fmt.Println(setupCmd.name) @@ -522,7 +515,7 @@ func TestLayers(t *testing.T) { } func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder) { - if err := imageBuilder.BuildImage(config, dockerfilesPath, dockerfile); err != nil { + if err := imageBuilder.BuildImage(t, config, dockerfilesPath, dockerfile); err != nil { t.Errorf("Error building image: %s", err) t.FailNow() } diff --git a/integration/integration_with_context_test.go b/integration/integration_with_context_test.go index 7f27fd17c7..30b17b675f 100644 --- a/integration/integration_with_context_test.go +++ b/integration/integration_with_context_test.go @@ -47,7 +47,7 @@ func TestWithContext(t *testing.T) { t.Parallel() if err := builder.BuildImageWithContext( - config, "", name, testDir, + t, config, "", name, testDir, ); err != nil { t.Fatal(err) } diff --git a/integration/k8s-job.yaml b/integration/k8s-job.yaml index c5809c4a28..4ebb628e8a 100644 --- a/integration/k8s-job.yaml +++ b/integration/k8s-job.yaml @@ -9,11 +9,12 @@ spec: containers: - name: kaniko image: localhost:5000/executor:latest + workingDir: /workspace args: [ "--context=dir:///workspace", "--destination={{.KanikoImage}}"] volumeMounts: - - name: context - mountPath: /workspace + - name: context + mountPath: /workspace restartPolicy: Never volumes: - name: context diff --git a/integration/k8s_test.go b/integration/k8s_test.go index d99c0a0ab6..935d5f9ef8 100644 --- a/integration/k8s_test.go +++ b/integration/k8s_test.go @@ -56,7 +56,7 @@ func TestK8s(t *testing.T) { t.Parallel() if err := builder.BuildDockerImage( - config.imageRepo, "", name, testDir, + t, config.imageRepo, "", name, testDir, ); err != nil { t.Fatal(err) } @@ -75,21 +75,21 @@ func TestK8s(t *testing.T) { t.Fatal(err) } - fmt.Printf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n", + t.Logf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n", testDir, kanikoImage) content, err := ioutil.ReadFile(tmpfile.Name()) if err != nil { log.Fatal(err) } - fmt.Printf("K8s template %s:\n%s\n", tmpfile.Name(), content) + t.Logf("K8s template %s:\n%s\n", tmpfile.Name(), content) kubeCmd := exec.Command("kubectl", "apply", "-f", tmpfile.Name()) RunCommand(kubeCmd, t) - fmt.Printf("Waiting for K8s kaniko build job to finish: %s\n", + t.Logf("Waiting for K8s kaniko build job to finish: %s\n", "job/kaniko-test-"+job.Name) - kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=120s", + kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=1m", "job/kaniko-test-"+job.Name) if out, errR := RunCommandWithoutTest(kubeWaitCmd); errR != nil { t.Log(kubeWaitCmd.Args) @@ -100,9 +100,25 @@ func TestK8s(t *testing.T) { t.Error(errD) } else { t.Log(string(outD)) - t.Error(errR) } - t.FailNow() + + descCmd = exec.Command("kubectl", "describe", "pods", "--selector", "job-name=kaniko-test-"+job.Name) + outD, errD = RunCommandWithoutTest(descCmd) + if errD != nil { + t.Error(errD) + } else { + t.Log(string(outD)) + } + + logsCmd := exec.Command("kubectl", "logs", "--all-containers", "job/kaniko-test-"+job.Name) + outL, errL := RunCommandWithoutTest(logsCmd) + if errL != nil { + t.Error(errL) + } else { + t.Log(string(outL)) + } + + t.Fatal(errR) } diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache") diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index 07ac75bdb4..caf1ab1127 100755 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -24,4 +24,4 @@ docker version echo "Running integration tests..." make out/executor make out/warmer -go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@" +go test ./integration/... --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@" diff --git a/scripts/minikube-setup.sh b/scripts/minikube-setup.sh index a92a0223a0..789fb1b5d4 100755 --- a/scripts/minikube-setup.sh +++ b/scripts/minikube-setup.sh @@ -15,6 +15,24 @@ set -ex -curl -Lo minikube https://storage.googleapis.com/minikube-builds/master/minikube-linux-amd64 -sudo install minikube /usr/local/bin/minikube -minikube start --profile=minikube --driver=docker \ No newline at end of file +# conntrack is required for minikube 1.19 and higher for none driver +if ! conntrack --version &>/dev/null; then + echo "WARNING: No contrack is not installed" + sudo apt-get update -qq + sudo apt-get -qq -y install conntrack +fi + +curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 +chmod +x minikube +sudo mv minikube /usr/local/bin/ + +sudo apt-get update +sudo apt-get install -y liblz4-tool +cat /proc/cpuinfo + +minikube start --vm-driver=none --force +minikube status +minikube addons enable registry +kubectl cluster-info + +kubectl port-forward --namespace kube-system service/registry 5000:80 & diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh index c8e9578430..38f0d84282 100755 --- a/scripts/travis-setup.sh +++ b/scripts/travis-setup.sh @@ -20,7 +20,6 @@ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubun sudo apt-get update sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && sudo mv container-diff-linux-amd64 /usr/local/bin/container-diff -docker run -d -p 5000:5000 --restart always --name registry registry:2 mkdir -p $HOME/.docker/ echo '{}' > $HOME/.docker/config.json