Skip to content

Commit

Permalink
Merge pull request #1124 from samos123/travis-k8s-test
Browse files Browse the repository at this point in the history
Travis k8s integration test
  • Loading branch information
samos123 authored Mar 16, 2020
2 parents 9cfb196 + 882793b commit 23e3fe7
Show file tree
Hide file tree
Showing 11 changed files with 266 additions and 79 deletions.
14 changes: 10 additions & 4 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ language: go
os: linux
dist: bionic
env:
- IMAGE_REPO=localhost:5000
global:
- IMAGE_REPO=localhost:5000 REGISTRY=localhost:5000
go:
- "1.13.3"
go_import_path: github.com/GoogleContainerTools/kaniko
Expand All @@ -21,11 +22,16 @@ jobs:
- make travis-setup
script:
- make integration-test-layers
- name: build-image-and-k8s-integration-test
before_install:
- make travis-setup
- make minikube-setup
script:
- make images
- make push
- make integration-test-k8s
- name: integration-test-misc
before_install:
- make travis-setup
script:
- make integration-test-misc
- stage: build-images
script:
- make images
28 changes: 18 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -55,28 +55,36 @@ out/warmer: $(GO_FILES)

.PHONY: travis-setup
travis-setup:
@ ./travis-setup.sh
@ ./scripts/travis-setup.sh

.PHONY: minikube-setup
minikube-setup:
@ ./scripts/minikube-setup.sh

.PHONY: test
test: out/executor
@ ./test.sh
@ ./scripts/test.sh

.PHONY: integration-test
integration-test:
@ ./integration-test.sh
@ ./scripts/integration-test.sh

.PHONY: integration-test-run
integration-test-run:
@ ./integration-test.sh -run "TestRun"
@ ./scripts/integration-test.sh -run "TestRun"

.PHONY: integration-test-layers
integration-test-layers:
@ ./integration-test.sh -run "TestLayers"
@ ./scripts/integration-test.sh -run "TestLayers"

.PHONY: integration-test-k8s
integration-test-k8s:
@ ./scripts/integration-test.sh -run "TestK8s"

.PHONY: integration-test-misc
integration-test-misc:
$(eval RUN_ARG=$(shell ./misc-integration-test.sh))
@ ./integration-test.sh -run "$(RUN_ARG)"
$(eval RUN_ARG=$(shell ./scripts/misc-integration-test.sh))
@ ./scripts/integration-test.sh -run "$(RUN_ARG)"

.PHONY: images
images:
Expand All @@ -86,6 +94,6 @@ images:

.PHONY: push
push:
docker push $(REGISTRY)/executor:latest
docker push $(REGISTRY)/executor:debug
docker push $(REGISTRY)/warmer:latest
docker push $(REGISTRY)/executor:latest
docker push $(REGISTRY)/executor:debug
docker push $(REGISTRY)/warmer:latest
40 changes: 0 additions & 40 deletions integration-test.sh

This file was deleted.

1 change: 1 addition & 0 deletions integration-test.sh
57 changes: 34 additions & 23 deletions integration/images.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,23 +185,8 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string {
return flags
}

// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
// The resulting image will be tagged with imageRepo. If the dockerfile will be built with
// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
_, ex, _, _ := runtime.Caller(0)
cwd := filepath.Dir(ex)

return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd)
}

func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error {
if _, present := d.filesBuilt[dockerfile]; present {
return nil
}
gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo

fmt.Printf("Building images for Dockerfile %s\n", dockerfile)
func (d *DockerFileBuilder) BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir string) error {
fmt.Printf("Building image for Dockerfile %s\n", dockerfile)

var buildArgs []string
buildArgFlag := "--build-arg"
Expand Down Expand Up @@ -230,13 +215,39 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
dockerCmd.Env = append(dockerCmd.Env, env...)
}

timer := timing.Start(dockerfile + "_docker")
out, err := RunCommandWithoutTest(dockerCmd)
timing.DefaultRun.Stop(timer)
if err != nil {
return fmt.Errorf("Failed to build image %s with docker command \"%s\": %s %s", dockerImage, dockerCmd.Args, err, string(out))
}
fmt.Printf("Build image for Dockerfile %s as %s. docker build output: %s \n", dockerfile, dockerImage, out)
return nil
}

// BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker.
// The resulting image will be tagged with imageRepo. If the dockerfile will be built with
// context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket.
func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error {
_, ex, _, _ := runtime.Caller(0)
cwd := filepath.Dir(ex)

return d.BuildImageWithContext(config, dockerfilesPath, dockerfile, cwd)
}

func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, dockerfilesPath, dockerfile, contextDir string) error {
if _, present := d.filesBuilt[dockerfile]; present {
return nil
}
gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo

var buildArgs []string
buildArgFlag := "--build-arg"
for _, arg := range argsMap[dockerfile] {
buildArgs = append(buildArgs, buildArgFlag, arg)
}

timer := timing.Start(dockerfile + "_docker")
d.BuildDockerImage(imageRepo, dockerfilesPath, dockerfile, contextDir)
timing.DefaultRun.Stop(timer)

contextFlag := "-c"
contextPath := buildContextPath
Expand Down Expand Up @@ -269,7 +280,7 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
}

// build kaniko image
additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
additionalFlags := append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
kanikoImage := GetKanikoImage(imageRepo, dockerfile)
fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags)

Expand Down Expand Up @@ -301,16 +312,16 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig,
kanikoCmd := exec.Command("docker", dockerRunFlags...)

timer = timing.Start(dockerfile + "_kaniko")
out, err = RunCommandWithoutTest(kanikoCmd)
out, err := RunCommandWithoutTest(kanikoCmd)
timing.DefaultRun.Stop(timer)

if err != nil {
return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out))
return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out))
}

if outputCheck := outputChecks[dockerfile]; outputCheck != nil {
if err := outputCheck(dockerfile, out); err != nil {
return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", dockerImage, kanikoCmd.Args, err, string(out))
return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out))
}
}

Expand Down
22 changes: 22 additions & 0 deletions integration/k8s-job.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
apiVersion: batch/v1
kind: Job
metadata:
name: kaniko-test-{{.Name}}
spec:
template:
spec:
hostNetwork: true
containers:
- name: kaniko
image: localhost:5000/executor:debug
args: [ "--context=dir:///workspace",
"--destination={{.KanikoImage}}"]
volumeMounts:
- name: context
mountPath: /workspace
restartPolicy: Never
volumes:
- name: context
hostPath:
path: {{.Context}}
backoffLimit: 1
106 changes: 106 additions & 0 deletions integration/k8s_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*
Copyright 2018 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"testing"
"text/template"
)

type K8sConfig struct {
KanikoImage string
Context string
Name string
}

func TestK8s(t *testing.T) {
cwd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}

dir := filepath.Join(cwd, "dockerfiles-with-context")

testDirs, err := ioutil.ReadDir(dir)
if err != nil {
t.Fatal(err)
}

builder := NewDockerFileBuilder()

for _, tdInfo := range testDirs {
name := tdInfo.Name()
testDir := filepath.Join(dir, name)

t.Run("test_k8s_with_context_"+name, func(t *testing.T) {
t.Parallel()

if err := builder.BuildDockerImage(
config.imageRepo, "", name, testDir,
); err != nil {
t.Fatal(err)
}

dockerImage := GetDockerImage(config.imageRepo, name)
kanikoImage := GetKanikoImage(config.imageRepo, name)

tmpfile, err := ioutil.TempFile("", "k8s-job-*.yaml")
if err != nil {
log.Fatal(err)
}
defer os.Remove(tmpfile.Name()) // clean up
tmpl := template.Must(template.ParseFiles("k8s-job.yaml"))
job := K8sConfig{KanikoImage: kanikoImage, Context: testDir, Name: name}
if err := tmpl.Execute(tmpfile, job); err != nil {
t.Fatal(err)
}

fmt.Printf("Testing K8s based Kaniko building of dockerfile %s and push to %s \n",
testDir, kanikoImage)
content, err := ioutil.ReadFile(tmpfile.Name())
if err != nil {
log.Fatal(err)
}
fmt.Printf("K8s template %s:\n%s\n", tmpfile.Name(), content)

kubeCmd := exec.Command("kubectl", "apply", "-f", tmpfile.Name())
RunCommand(kubeCmd, t)

fmt.Printf("Waiting for K8s kaniko build job to finish: %s\n",
"job/kaniko-test-"+job.Name)

kubeWaitCmd := exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=60s",
"job/kaniko-test-"+job.Name)
RunCommand(kubeWaitCmd, t)

diff := containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--no-cache")

expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage)
checkContainerDiffOutput(t, diff, expected)
})
}

if err := logBenchmarks("benchmark"); err != nil {
t.Logf("Failed to create benchmark file: %v", err)
}
}
40 changes: 40 additions & 0 deletions scripts/integration-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/bin/bash
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -ex

GCS_BUCKET="${GCS_BUCKET:-gs://kaniko-test-bucket}"
IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}"

docker version

# Sets up a kokoro (Google internal integration testing tool) environment
if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then
echo "Installing dependencies..."
source "$KOKORO_GFILE_DIR/common.sh"
mkdir -p /usr/local/go/src/github.com/GoogleContainerTools/
cp -r github/kaniko /usr/local/go/src/github.com/GoogleContainerTools/
pushd /usr/local/go/src/github.com/GoogleContainerTools/kaniko
echo "Installing container-diff..."
mv $KOKORO_GFILE_DIR/container-diff-linux-amd64 $KOKORO_GFILE_DIR/container-diff
chmod +x $KOKORO_GFILE_DIR/container-diff
export PATH=$PATH:$KOKORO_GFILE_DIR
cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json
fi

echo "Running integration tests..."
make out/executor
make out/warmer
go test ./integration/... -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 50m "$@"
Loading

0 comments on commit 23e3fe7

Please sign in to comment.